{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "EMNIST NLP",
      "provenance": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyMohBCMz+k1RkZr6+lDvr04",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wandb/awesome-dl-projects/blob/master/ml-tutorial/EMNIST_NLP.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LHCCGutqrj-r",
        "colab_type": "text"
      },
      "source": [
        "# Imports and Setups"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qiFYfrVpXbAP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        },
        "outputId": "a3c7303a-d432-40a8-a432-e66ff22b75a2"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue Sep  8 03:18:38 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 450.66       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   48C    P8    10W /  70W |      0MiB / 15079MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eGY8OiyGpN-x",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        },
        "outputId": "39661dcd-a446-4c14-b39b-4eff0609a379"
      },
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "\n",
        "from tensorflow.keras.layers import *\n",
        "from tensorflow.keras.models import *\n",
        "import tensorflow_datasets as tfds\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "%matplotlib inline\n",
        "tfds.disable_progress_bar()\n",
        "\n",
        "import csv\n",
        "import string\n",
        "import pathlib\n",
        "import gzip\n",
        "from tqdm import tqdm\n",
        "import seaborn as sns\n",
        "from itertools import groupby\n",
        "\n",
        "from sklearn.utils import shuffle"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.3.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HqUfAaLR7H4F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install wandb"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3GeVDMnA7LOr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "9786d665-f901-4ac5-83c3-58862bb3ff5b"
      },
      "source": [
        "import wandb\n",
        "from wandb.keras import WandbCallback\n",
        "\n",
        "wandb.login()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://app.wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9VJbZx9g9Uz",
        "colab_type": "text"
      },
      "source": [
        "# Download Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WHw1hCpYg_1R",
        "colab_type": "text"
      },
      "source": [
        "#### Wiki-split dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4SOyIXEHnI4p",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "e49ea0c0-8bfb-47c2-ab1d-ebd2ba97bedf"
      },
      "source": [
        "!git clone https://github.com/google-research-datasets/wiki-split"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'wiki-split'...\n",
            "remote: Enumerating objects: 32, done.\u001b[K\n",
            "remote: Total 32 (delta 0), reused 0 (delta 0), pack-reused 32\u001b[K\n",
            "Unpacking objects: 100% (32/32), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ewq14NyxhId2",
        "colab_type": "text"
      },
      "source": [
        "> Side Note: We will be using `test.tsv` for our experiment to speed up the process."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gcRbRClXjBq1",
        "colab_type": "text"
      },
      "source": [
        "#### EMNIST/Bymerge Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0qrzBv-XjFiQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 148
        },
        "outputId": "6f6348f1-3876-4ca3-ce42-7fd54e38e2cd"
      },
      "source": [
        "# Gather EMNIST/bymerge dataset\n",
        "train_ds, validation_ds = tfds.load(\n",
        "    \"emnist/bymerge\",\n",
        "    split=[\"train[:85%]\", \"train[85%:]\"],\n",
        "    as_supervised=True\n",
        ")"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDownloading and preparing dataset emnist/bymerge/3.0.0 (download: 535.73 MiB, generated: Unknown size, total: 535.73 MiB) to /root/tensorflow_datasets/emnist/bymerge/3.0.0...\u001b[0m\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/urllib3/connectionpool.py:847: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
            "  InsecureRequestWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Shuffling and writing examples to /root/tensorflow_datasets/emnist/bymerge/3.0.0.incompleteYVGGB4/emnist-train.tfrecord\n",
            "Shuffling and writing examples to /root/tensorflow_datasets/emnist/bymerge/3.0.0.incompleteYVGGB4/emnist-test.tfrecord\n",
            "\u001b[1mDataset emnist downloaded and prepared to /root/tensorflow_datasets/emnist/bymerge/3.0.0. Subsequent calls will reuse this data.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Of-n4NypN_4",
        "colab_type": "text"
      },
      "source": [
        "# Load Data, Clean it and preprocess"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Av4Db7lpN_8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "9c180f3b-3a38-427d-a3e0-0d99eee0c759"
      },
      "source": [
        "raw_sentences = []  # empty list to store sentences\n",
        "with open(\"wiki-split/test.tsv\", \"r\") as f:\n",
        "    reader = csv.reader(f, delimiter='\\t')  # read a tsv file\n",
        "    for row in tqdm(reader):\n",
        "        raw_sentences.extend(row[1].split(\"<::::>\"))\n",
        "print(\"Total Sentences: \", len(raw_sentences))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5000it [00:00, 155032.23it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Total Sentences:  10000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0C-5foj6pOAN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "outputId": "d7225589-cc49-483a-b566-022184aa432e"
      },
      "source": [
        "# Some samples\n",
        "print(raw_sentences[99], '\\n', raw_sentences[100])"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " He was was arrested and booked on charges of first - degree murder and first - degree robbery . \n",
            " A 2006 estimate by the International Organization for Migration put the number of Sudanese people in the UK at a much higher figure . \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mIPqzykIkXxd",
        "colab_type": "text"
      },
      "source": [
        "#### Remove punctuations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-HtS5z3MpOAe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "12d35e33-003d-4913-f352-76f62dd8466c"
      },
      "source": [
        "# As we see there are lots of punctuations which we dont have in EMNIST, so we are going to remove them, \n",
        "# and replace multiple spaces with one\n",
        "import re\n",
        "\n",
        "sentences = []\n",
        "table = str.maketrans({key: None for key in string.punctuation})  # translation table\n",
        "\n",
        "for sentence in raw_sentences:\n",
        "    # remove punctuation and non-ascii characters\n",
        "    clean_sentence = re.sub('  +', ' ', sentence.translate(table)).\\\n",
        "                        encode(\"ascii\", 'ignore').decode()  \n",
        "    sentences.append(clean_sentence.strip())  # add to clean sentences\n",
        "    \n",
        "print(sentences[99], '\\n', raw_sentences[99])  # to verify"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "He was was arrested and booked on charges of first degree murder and first degree robbery \n",
            "  He was was arrested and booked on charges of first - degree murder and first - degree robbery .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DxGdQ087p1V",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e87280c8-2c8f-4db9-9f50-5be1a2e0a99e"
      },
      "source": [
        "print('Number of sentences', len(sentences))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of sentences 10000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lwUb0KtemeK3",
        "colab_type": "text"
      },
      "source": [
        "#### EMNIST Labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oSXgQqsOEF38",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "1fa42c4c-c317-46f7-da47-c7fa4068d6ae"
      },
      "source": [
        "MAPPINGS = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', \n",
        "          'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',\n",
        "          'a', 'b', 'd', 'e', 'f', 'g', 'h', 'n', 'q', 'r', 't']\n",
        "\n",
        "print(len(MAPPINGS))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "47\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "USfu5WfonDPZ",
        "colab_type": "text"
      },
      "source": [
        "#### Dataloader for EMNIST"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WE7sVKVPEJzK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "AUTO = tf.data.experimental.AUTOTUNE\n",
        "BATCH_SIZE = 256\n",
        "\n",
        "## We are transposing to rotate the image by 90 deg clockwise making the images human friendly.\n",
        "def transpose(image, label):\n",
        "  image = tf.image.convert_image_dtype(image, dtype=tf.float32) # scale image pixels to [0,1]\n",
        "  image = tf.transpose(image, [1,0,2]) # transpose to get human friendly image, since rotation\n",
        "  return image, label\n",
        "\n",
        "trainloader = (\n",
        "    train_ds\n",
        "    .shuffle(1024)\n",
        "    .map(transpose, num_parallel_calls=AUTO)\n",
        "    .prefetch(AUTO)\n",
        ")"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Exo6ceZ1EJv0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 591
        },
        "outputId": "7148060a-9806-4749-d594-dccd05aa6a9f"
      },
      "source": [
        "plt.figure(figsize=(10, 10))\n",
        "n = 0\n",
        "for img, label in trainloader.take(25):\n",
        "    ax = plt.subplot(5, 5, n+1)\n",
        "    plt.imshow(tf.reshape(img, (28,28)), cmap='gray')\n",
        "    plt.title(MAPPINGS[int(label.numpy())])\n",
        "    plt.axis('off')\n",
        "    n+=1"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAAI+CAYAAABe7hvVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhU1bU+/nerKJPMk8gMgoAMoomighghDjjrvThrBn/GKWr0q8mVOGtikqsmajTxmoioQU0EBEVFVEAEoyIqIPOMzIIoIk7n90fjyrs3XWVRXdWna/f7eZ48dxVn0xzr1Kk+d6+91nZJkkBEREQkNrukfQIiIiIixaCHHBEREYmSHnJEREQkSnrIERERkSjpIUdERESipIccERERiZIeckRERCRK0T7kOOf2cc597px7NO1zkYpxzp3unPvAObfFObfQOdcv7XOSneOcS5xznYI/u1H3Z+lxzu3hnHvIObfUOfeJc26Gc+6YtM9L8uec6+qce9k597FzboFz7uS0z6lQon3IAXAfgDfTPgmpGOfcIAB3APgRgD0B9AewKNWTEqnedgOwHMDhAOoDGArgSedcuxTPSfLknNsNwGgAYwE0AvD/AXjUOdc51RMrkCgfcpxzpwPYBGBC2uciFXYTgJuTJJmWJMk3SZKsTJJkZdonJVJdJUmyJUmSG5MkWbL9nhwLYDGAA9I+N8nLvgBaArgrSZKvkyR5GcAUAOeke1qFEd1DjnOuHoCbAfwi7XORinHO7QrgQABNt0+hrnDO3eucq5X2uYlIGedccwCdAcxK+1ykYByA/dI+iUKI7iEHwC0AHkqSZEXaJyIV1hxADQCnAegHoDeA/VE2PS4iKXPO1QDwGIBhSZLMSft8JC9zAawF8P+cczWccz9EWSqydrqnVRhRPeQ453oDGAjgrrTPRQpi6/b/e0+SJKuSJFkP4E4Ax6Z4TpKfr1H2wMpqAPgyhXORAnDO7QJgOIAvAFya8ulInpIk+RLASQAGA1gN4CoATwKIYqJgt7RPoMAGAGgHYJlzDgDqAtjVOdctSZI+KZ6X5CFJko3OuRUAEv7jtM5HKmQZyu7ND+jP2gOYl8rZSIW4si/Yh1A223rs9l+UUqKSJHkPZbM3AADn3OsAhqV3RoUT20POXwGMoNdXo+yL9aJUzkYK4e8ALnPOPY+y/6//SpRVAUhpeQLAUOfc+wA+BPADAMcD6JvqWUm+7gfQFcDAJEm2ftdgqdqccz1R9v9w7ALgYgB7AXg4zXMqlKgecpIk+QzAZ9++ds59CuDzJEnWpXdWUkG3AGiCshvwc5RNo96W6hlJPm7e/r/XADQEsBDAWUmSzEz1rGSnOefaArgQwDYAq7fPmgPAhUmSPJbaiUlFnAPgpyhLIU8GMChJkm3pnlJhuCTR7L+IiIjEJ6qFxyIiIiLf0kOOiIiIREkPOSIiIhIlPeSIiIhIlPSQIyIiIlHKWkLunFPpVQqSJHHfPWrn6XqmoxjXU9cyHbo346J7Mx6ZrqVmckRERCRKesgRERGRKEXV8bjQdtnlP8+A1NUTX3/9dRqnIyJS5fB3o5rLSlWjmRwRERGJkh5yREREJEp6yBEREZEoaU1OFscee6zFHTt2tHj06NHeuKVLl1qsnPSOGjVqlPH1V199ZfHKlSu9cV9++WVxT0xEcrLbbv6vilatWlncqVMni5cvX+6N27btPxtZr1q1qtw/l8rFa03Le52LGjVqWNy8eXPvGH9WNm3aZPGGDRu8cZX1u1IzOSIiIhIlPeSIiIhIlJSuIrVq1fJe33zzzRbzlGyvXr28cddcc43F69evL9LZlRaeAg3fr4MPPtjiTz75xOLHH3/cG/fRRx8V6eyk1IRT6ly2zMIp8G+++aZo5xSjmjVrWty5c2eLBw0a5I3r16+fxQcccIDFn332mTfu448/tviuu+6yeOTIkd64zz//PM8zlkx23XVXixs3bmzxoYce6o3bb7/9LM41dbXnnntafPjhh3vH6tata/HkyZMtHjp0qDdu7dq1Of1bFaWZHBEREYmSHnJEREQkStU+XcWrxPv06eMd23fffS3mady+fft64xo0aGBxdU1X8dQo4E+PXnTRRd6xY445xuJly5ZZ/Pzzz3vjlK6q3vjerFevnneMp8TZp59+6r3evHmzxarWK8OpvrZt23rHzj33XItPOeUUi7m6FPCvDac4sqUVb7rpJovD6/Tss89arBRjfmrXru295hTj2WefbTGnGgGgYcOGFmdKA2cTVt5l+tnjxo3zjnGVcjGvuWZyREREJEp6yBEREZEo6SFHREREolQt1+Tw+pFu3bpZ/Nvf/tYbx+twWD4dImPHa3AAP+/bv39/7xi/f1xyyt2PpXriz0b9+vUt7tGjhzeue/fu5f79WbNmea/ff/99i8M1XtVl7QevnwH877yrr77aO3bSSSdZzN9/3NUd8NdT8Pqa8F7n7wFe/9O7d29vHK/XqC7XpRCaNGli8YUXXugd+9GPfmQxv/fh+sl8ZFu7w20ceE0Ol6oDwJgxYyzWmhwRERGRnaSHHBEREYlStUhXhdNzAwYMsPjSSy+1+MADD6ysU4oCT1nutdde3jFOL3B3TMDvcrx48WKLt2zZUuhTlBzlOv1cbLvvvrvFLVq0sPiQQw7xxh1xxBHl/n1OcQHAunXrLA7LlmPuspspJQ8A9957r8XcrRgAvv76a4vHjh1r8cMPP+yNe+mllyzOlmrg7rr8GVPKPz/hEorBgwdbzL/LgB03zvxWuCxg48aNFvN3cNimIVySkAv+t8L7r7LokyYiIiJR0kOOiIiIRCmqdBVP0fJq8nPOOccbd95555U7LuyIylNt2bo6Vid77LGHxZyiCqs0OL3AKQjA37TtiSeesFgdjisXV93w5rRh6mrbtm0Wf/HFF96xilZFhGkLTm3us88+FofVOJyC4XRa+BmaPXu2xatXr/aO8X9LDBU9nMo46qijLOYqG8BPUa1atco79sgjj1g8fPhwi8PqKk5rhdVbUlh8j4QbpV533XUWN2vWzDvG98WSJUssfvrpp71xr7/+usXLly+3OKyGuvzyyy3m+y/b9efP18SJE71jlVVNq5kcERERiZIeckRERCRKesgRERGRKJX0QpNcS8MHDhzojatTp47FW7dutZg7MALAwQcfbHGbNm0qdK6x4HU4J598ssXcKRXwd8QNd2bndTi8Pofz/FJ44foXLhHt0KFDuX8O+NdvzZo13rGPP/7Y4nx2+Q7Pie9NPqdwF+wGDRpYzLn9cF1C06ZNLQ7Lb/PZcbkqCb//eB3O73//e4vD9g5z5861+A9/+IN3bOTIkRZzN/Js+FqEnaj5+vLaLt4dXrLj9zBcm8ZrSsPPM7fnuP766y0eNWqUN45/B/LaNO4WDvjrev72t79Z3K5dO28c/4wZM2ZYzOt9KpNmckRERCRKesgRERGRKJVcuoqnaNu3b+8d4xQVd4IMy795eo7L6W677TZvHG9CJ2W4HLxevXrl/jngT51yh2MAWLRoUcZjUjzhdDaXa3OH6jDlwOmqf//7396x9957z2LunJpvSTbfq5w24zQW4H8P8GeIOxyHr8MOx5XZybkYePNDABgyZIjFnLYL0xOcopozZ453jFNKuWrdurXFYTqFUy1VoZy4FHGJdtjRm+/p8D3k685xrp3lw88CX79sqWm+93nD3E2bNuX07xaaZnJEREQkSnrIERERkSjpIUdERESiVBJrcjK1Kz///PO9cXyMc/Zr1671xo0bN87iW2+91eK0StyqsrBMtWfPnuXGYSkwr3uaNGmSd2zZsmUW57MGQPITrsnhdVRcZsytEwB/LQtfV8AvU+Vy8nzX5PDniM83PHfekoHvb14DAAALFy60ONwFudS3cgjPn9sxvPnmmxZzWTjgb9GQ77ok/k4+/vjjLW7ZsqU3jttCTJ8+3WJ91+aOdxM//PDDvWO8hi1ck8Nl+ny/hN/Vme65cM1Xv379Mh7LhD+jaa2B00yOiIiIREkPOSIiIhKlKpmu4m65gN9Z94YbbrA4LCHnaTcuU+aUFAA8//zzFvOuxWFHVO2u63fUBICzzjrL4r59+1rM5cMAMHXqVItvv/1279iGDRssLvUy3lISpnw4Fcmf/UaNGnnjeMqZu9sCO7ZnKCQ+3/Bzwqmn+fPnWxymq7hDcz4dmauycMf1v/71rxZnKy0uBE5Lcbpqjz328MZx+4Enn3zS4vDcJTO+x7htRyi8vxs3bmxx586dy40Bf7dx/rfCXcgPPfTQcn92iO+zqtDZWjM5IiIiEiU95IiIiEiUqky6iqfOr7zySu/Yj3/8Y4t5M7BsG5LddNNNFj/33HPeuFatWll85plnlvvnwI4b21UXnPYLu99yZ1zuvsndMAFgwYIFGY8pRZWOMLXEHYW5+3HYvZrTHdkqM/IRVu/xz+M4/Mxw11beOJDT1IA/XR77BrDF/O8LPzt9+vSxmDseh9/JnMbm7tixX4tC4vsvW/onvJf4d9vRRx9tcdg1mVPQfP3yvdczdbZOq6u1ZnJEREQkSnrIERERkSjpIUdERESilNqanDB3y+XgvJs44Hd8ZOFuqk899ZTFvCbksssu88ZxSfo+++xjcVgyXszy2KqMc6/hrsKcf+f3JywJXbFihcXqapwevpa8Bgfw17dxHI4rdBko32dhSTqvg+Nj4b3IHVz5/MKuxtrdOn/82eFutwBw4YUXWswtB8L3O9Mu9ZK71atXWzxmzBjvWJcuXSwO71u+LmFbiEIK11dVtc7WmskRERGRKOkhR0RERKJUqfkYTlHx5o4A8PDDD1ucKT0FAJ988onFb7zxhnfspJNOsvjnP/+5xWEXTi5H5W6uYQqN8bj333/fO8YbE8YmLBvk94inKV999VVvHG/KmW/KgEsiOcXRokULb1w+aUU+p7DEnVMhpV7uztcvLB3ljqbcKiBMIXFaONvnIdv9w9eIU54nnHCCN45TyTwVH25G+e6771rM92OYNlWpcv74/gvTVfyarzuXDAPAX/7yF4u507nkjjfIfeSRRzKO487TgN+VmNNV4T3CHYr5WoYprkwpr/C6PvHEExn/rTRoJkdERESipIccERERiVKlpqt4s8err77aO7bvvvtaHE57c8qAO7MeccQRGf8tnt5et26dd+y1116zmDf0a9iwoTfu4osvLvfnhZsAbtq0KeN5VBdhpVtY5ZIJp6H23ntv79iBBx5oMadTjjvuOG9ctk3rGF9DTnved9993rgZM2ZYPHv2bO9YKW/ymC3VlKnTMOCnmvj+C1/zJp/hPcwpY572DruM82tOr3GFCQAsXbrUYu54HH7uwjSXZMfXjb+vDzjgAG8cfya2bt1q8bhx47xxkydPtljXouL4sw4Av/vd7ywOU1l8n3GHaq5+Avx7hu/hcOeBs88+22L+nPDvU8C/5lUhXayZHBEREYmSHnJEREQkSnrIERERkSgVfU0OlyGeeOKJ5caAn7MPy3a53JfXRISlv7yWgstKuesmAEyZMsVi7sLZsWNHb9zPfvYzlCfMLZd6mXEh1KlTx3sddt9knM/ndVBhOXH//v0t7tSpk8XcpRrYsVN1JnyduEycWw8A/rqQlStXesfWr1+f079VysL1NLzuhq8D4Of6eS1P2CWZ101xF20uEwf8dTj83RGuheI1VbymQB2OK6Zp06YWX3fddRYPHDjQG8fXY9SoURb/61//8sbxfcbCdV98rbmFQfi9wt/56qTul5eH63X4NbdcCNfJ1KpVy2L+bt1///29cZnux7ClSlXrbK2ZHBEREYmSHnJEREQkSkVPV3GKYOHChRbzBpqAP03G4wDgpZdespjTB2F3Td4MjMu6w+m5TKWM2dJOPA2ea3l07Hj6csCAAd4xvk5z5szxjvHUN5eJX3DBBd44TmXx+79mzRpvXK5l3ZzW4nTaMccc44076KCDLA5LyEePHm1xqZXEZkuzcpwtXRV2Kmddu3a1OLxH+P3mlFf48/jf4vs2/HmcDuO0hVLHO4dLhgH/XuA4HDdz5kyL77rrLos//PBDbxynv/h+5pYQgP856NGjh8Vhi4G7777b4qeffto7lik1JtlLufl3L3cc79y5c8a/wympcDlIVSgbZ5rJERERkSjpIUdERESiVPR0FU+Rjx8/3uJly5Z5437wgx9Y/PLLL3vH5s+fbzGnJgpRScGr/Lt165bxGK9iDytsYp4iD6tkODVQu3Zti8PN29q1a2cxTz8DwNChQy3m6hpe5Q/4m35OmzbNYq7mAHLfIJXPkSsHbrnlFm8cT6sffPDB3jHu6MqfiVIQ3i98bTkO0398Xfbaay/vGKeh+DqHU9ac2uSUVFiFx/ccp5wXLVrkjePX3G271FKIaeBrcdRRR3nHuKKKU03h+8qVrJymDDeJ5M8Ebwgbdpfniir+DISf2SOPPNLi559/3jumdFVuwg2NBw0aZDFfvzBFyfc0dzXmamWg6t2DmskRERGRKOkhR0RERKKkhxwRERGJUqXuQs7rOcKyM97ZuzJL0Dj/yznj8Bh33mzTpo03Liy5LXWcUw1L/bl8m3cND3cC5/fysMMO8441btzYYs77hmtBeB3OCy+8YHGuO4OH14XX7vD5hp1Tec1IkyZNvGN8vqVQuszXMizD5lYNXBLM6yMAoGXLlhbvvvvu3jEeG/69TPi9CtdcfPTRRxbzZ++dd97xxvE6Pe5+XNXWA1QF4X3Qvn17i88///yMx/jv8ToeABg8eLDFXGrOHavL+3vfCr/jef0VtwIJ12fymrhwvaDkJlw/2bdvX4tbt26d8e/xvfnEE09YvGHDhgKeXeFpJkdERESipIccERERiVKlpquySatLInfBDadaWXXqeMxT/hMmTPCODR8+3GLe2DIsE+fNNXkTR8BP/XE5Y9jJmKeqOZ2ZrcMxT7GHU6/nnHOOxVxCHqZZeOqcy9gB4LPPPrO4qqaoGF9LTusAfoqKU3QhvrbhVHemcvAwTcH3N6cZwms+d+5ci7lMmVOXALB69WqLVTqcHZeCA36ZeFhCzteN76Xws86fAz4Wpn459ZRpA2XAX77Am0mGmzDzz1dqMne8AXbY4Z2XE4T3N+NrOX36dIurWofjkGZyREREJEp6yBEREZEoVZl0VVq4QogrBkIrVqyw+LnnnvOOFaLzclXF6RkAGDZsmMXz5s2z+JprrvHG8fsaVihlmgbnqivA3ywuW7dU/nlcEde7d29vHHfV5unbJUuWeOO4gydXdQGlnRoJz503U5w6darF2dJaLVq08I516NDBYu5yzd2wAf9zxN2K+WeHr/nzFW78yJVySlvsiKsAw/REto03Gd+bYUqCK204jTFmzBhvHL/OtIFy+PN1PQuPO5Vfcskl3jFOZ/J3afh7jVP3YRqxKtNMjoiIiERJDzkiIiISJT3kiIiISJSq/Zocxus5sqnqJXPFxKW7vI5j7Nix3jjevfvwww/3joW74H4r7KbLP4PX5IQ7YWe6bmFpOLcL4FLUN954wxv34osvWpzrDuelICwD5vdg5cqVFoedZLN1Q+YOufmsyeGuy4BfUs7nEZYma93Gjvg+4N26uWQc2LGknPFnZN26dRbzrtOA3/GWy4nDtVOff/75d522VAL+zg1bpWTq2B/ec/wdka2NR1WjmRwRERGJkh5yREREJEpKV8lO4elnLr2+5557vHFcbrhx40bvGHdA5o65YSqkZ8+e5Z5DmJ7KtOEjb+IIAI888ojFnHb717/+5Y3jY+GUbUwydaoNp6K53DcsHeWy4LffftviMCXJ12XLli0Wh93Ducy9OqeFK4pTs9xhPBS+x4sXL7b4tttus5g3xgT8TRl1nUpLtlQvHws3Rx09erTFpdQ2RTM5IiIiEiU95IiIiEiU9JAjIiIiUar2a3I4t8glcgDQtm1bi3ldQinlIyvL2rVrvdevvPKKxbzDMODvDs7vMZeJA7mX9HMemdd4hDuo8xodXnei6+kLc/b8OnyveC1PuPYql5+vUvDC4ffytddes/jee+/1xvF2KeGWJo899pjFvKWJSsFLG//+mjRpUsZx/BkKvz+XLVtW+BOrBJrJERERkSjpIUdERESi5MIuqN5B5zIfjASXWnbp0sU7dtJJJ1n87rvvWswdcYHClxknSVJ+C8oKSut6hmkn3gG8bt26FoedOPPB6RQuBQfSm3IvxvWsDvdmVVSK92a40zjvJM/l/IC/u3h1KA2vLvcmdzVu3Lixdyxs3fEt/iyU97qqyXQtNZMjIiIiUdJDjoiIiESp2qersuGurZVZEVKKU+KSWXWZEq8OdG/GRfdmPJSuEhERkWpFDzkiIiISJT3kiIiISJSqfcfjbNQJV0REpHRpJkdERESipIccERERiVLWEnIRERGRUqWZHBEREYmSHnJEREQkSnrIERERkShF+5DjnNvHOfe5c+7RtM9F8uOcO985975z7jPn3Grn3P3OufK3zJWS4Jw73Tn3gXNui3NuoXOuX9rnJDvPObfEOTcw7fOQinPOXeqce8s5t80593Da51No0T7kALgPwJtpn4Tkxzl3FYA7APw/APUBHAygLYDxzrnd0zw3yY9zbhDKrumPAOwJoD+ARamelIh8COBWAH9L+0SKIcqHHOfc6QA2AZiQ9rnIznPO1QNwE4DLkiR5PkmSL5MkWQLgvwG0A3B2iqcn+bsJwM1JkkxLkuSbJElWJkmyMu2TEqnOkiR5OkmSUQA2pH0uxRDdQ872X5A3A/hF2ucieTsEQE0AT/MfJknyKYDnAAxK46Qkf865XQEcCKCpc26Bc26Fc+5e51yttM9NROIV3UMOgFsAPJQkyYq0T0Ty1gTA+iRJyttXY9X241JamgOoAeA0AP0A9AawP4ChaZ6UiMQtqocc51xvAAMB3JX2uUiFrAfQxDlX3t5qe20/LqVl6/b/e0+SJKuSJFkP4E4Ax6Z4TiISudg26ByAsjUby5xzAFAXwK7OuW5JkvRJ8bxk50wFsA3AKQCe/PYPnXN1ARwD4H9SOi/JU5IkG51zKwBwi3W1WxeRoortIeevAEbQ66tR9tBzUSpnI3lJkuRj59xNAO5xzm1G2QLyvQH8GcAKAMPTPD/J298BXOacex7AlwCuBDA23VOSCqjhnKtJr7/KkGKWKmz7jPluAHZF2aRATUR0LaNKVyVJ8lmSJKu//R+ATwF8niTJurTPTXZOkiS/Q9mMzR8AbAbwBoDlAI5MkmRbmucmebsFZW0d5gH4AMA7AG5L9YykIp5DWRry2//dmOrZSL6Gouz6/RJllatbEdFaOW3QKSIiIlGKaiZHRERE5Ft6yBEREZEo6SFHREREoqSHHBEREYmSHnJEREQkSln75DjnVHqVgiRJXDF+rq5nOopxPXUt06F7My66N+OR6VpqJkdERESiFFvH44LaZZddyo2/+eYbb1z4WkQqrkmT/+zD2qBBg4zjli5davGXX35Z1HOSHe26664Wf/311ymeiciONJMjIiIiUdJDjoiIiERJDzkiIiISJa3JIXvssYf3+oc//KHFvXv3tnjSpEneuMmTJ1us9TnF4dx/Fs43btzYO1a3bt1y/87q1au917xeQ2sHqqaOHTta/Pzzz1vcpk0bb9yaNWss7tevn8W8PkeKh78rzz77bIsnTJjgjVuyZEllnZLkqVGjRuXGH330kTcufF0qNJMjIiIiUdJDjoiIiESp2qeratasafGQIUO8Y3fccYfFTZs2tfjqq6/2xk2ZMsVipasKh69Nly5dLL7iiiu8cd27d7eY01pjx471xs2cOdNiTjGuW7fOG5ck6uVVWbhMHPBTVB06dLB40aJF3rhjjjnGYqWoKl/btm0tvueeeywO77lLLrnE4vA+k3Tw9yoAXHjhhRafccYZFv/1r3/1xj3wwAMWf/XVV0U6u8LTTI6IiIhESQ85IiIiEqVqn64aNGiQxdddd513jKfSuUpg5MiR3rhSmrqraji9tPvuu3vHOnfubPHJJ59sMVfTAEDz5s0t5s7UNWrU8Mbtt99+FnOl1QsvvOCN27ZtW07nLhU3YMAA73X79u0tXrx4scU8jQ4ACxYsKOp5iY/vK8BPEfN926NHD29c/fr1LVa6qmpo2bKl9/rUU0+1mJcFhFWspUozOSIiIhIlPeSIiIhIlPSQIyIiIlGq9mtyDjjgAIu5LBLwS4nHjx9v8bJly4p/YhHjXYt5DcZBBx3kjeNSfc4VhyWQmfTs2dN7zesIuLNuWII8f/58iz/77LOc/i3JHXfLPeuss7xjvPZj+vTpFqtMPF18zwL+vcXXLFy7I1UDX5devXp5x7ibOK+RjIU+kSIiIhIlPeSIiIhIlKpMuqpdu3YWc7kw4Jf7Pv744xbnumFYOIU6ePBgiy+++GKLd9vNfzu4bHzixIkWq6vxd+Npz3Dj06OOOsri888/3+IwvcSfiWzTqJmOhZ2Lecq9a9euFocdrLnr7j//+U/vmMrL88PX6PTTT7f4+OOP98atXbvW4l/+8pcWq/w4XXvvvbf3+rjjjkvpTCQf/DswLPNv0KBBZZ9OpdJMjoiIiERJDzkiIiISpdTSVWFqiFNUN954o3dsxYoVFnMqIdd0Va1atbzX/fv3t5i7Gn/++efeuOHDh1s8evTonP4tKcMVUGGHzZNOOsnivn37WszdUYHMaajwz/PZUJO7tIZVXdzBmqvqAD+dIrnjKfJ7773XYk5FA8BVV11lcbgpp6Qn/L6uV69eueM2b97svVY3+KqBvzPDSrlMYtmoWDM5IiIiEiU95IiIiEiU9JAjIiIiUaoyJeSc4w3X0IT54Fxw2fIJJ5zgHTvllFPK/Tvz5s3zXj/99NMWq/NtdmGZOK+7OfbYY71j/P6H1zoX2XLFvK4qXO/B/xZ/prjrMuB/FseOHesdGzVqlMVff/11jmdc/YSfh5tuusniOnXqWPz2229745544oninpjkJds6Dl53w602AGDNmjVFOyfJjq9Zw4YNLd5vv/28cVxezutc33//fW9cqbZO0UyOiIiIREkPOSIiIhKl1NJVXLoNAI0aNco4lqdDW7VqZfHChQu9cZzG4BTETx4M8GEAACAASURBVH7yE28cd9LlkuC//e1v3rjZs2eXez5hB2Uuz6tOKQz+79533329Y9dcc43FnTp18o5lSlFlKw3nOOx+u3r1aovHjBljMW88BwADBgywuHXr1haHU/GNGze2eMiQId6xSZMmWbxhwwaLS3Uqt5AydTUG/M7GfM+dccYZ3jiVHFcd3AYivJ58/3BaeOXKld64MGUslYe/x/r162fxoYce6o3j+5Y3wn333Xe9caX6HaeZHBEREYmSHnJEREQkSnrIERERkShV6pocXvtw8803e8e45DjMy//ud7+z+L333rM4LCXmstVrr73W4t69e2c8pwcffNDiESNGeMf4PPhnhyXRLVq0sDgsgc1164lSwdeQc75XXHGFN47X6ITlxJmE15Nfc4v/W2+91Rs3Y8YMizdt2mTxnXfe6Y3jNQbZ8H9juGMvbz2xceNGi0s1X11IvA6Ot24A/Lz//fffb/GCBQuKf2KSl86dO1vM2+4A/r3E38kTJkzwxmmNVeUJ14ry2hteUxWuh+W1hffdd5/F4fqqUqWZHBEREYmSHnJEREQkSkVPV/E0NU9nh12IeQot3H34n//8p8Vbtmwp92cDforktNNOs5g7rAJ+V9ynnnrKYp62C/8ep9N+//vfe+MWL15s8bhx47xjsaWrOPW01157WRymBHmX7xBft1y7F0+bNs3iKVOmeONWrVplcceOHcuNAWDPPffM+G9lEk4Bi4/bMWTqagz45ajh/SNVB3cCP/LIIy3eZ599Mv6d8ePHWzx//vzinJh8p3BZwCGHHGLx/vvvb3HYMoNT/FOnTrV427ZthT7FVOgbXERERKKkhxwRERGJUsHTVWEKqWfPnhY//PDDFjdr1swbx52CJ0+e7B3jipa6detaPGjQIG/cDTfcYHE4Xc54s83/+q//svj666/3xnEKpmXLlhbPmTPHG3fxxRdbzB0jYxBOgXKVxdFHH21x2PE4/BywTJ2Mw+lR3iD1xhtvtDhMZ/L0K38mwin2TNVV2Totiy98D++66y6LM3U1Bvz7LJ/NbrNVxnFaUyqGO89zdU74PcBVU1yVunXr1iKenYT4u4ur4QDghz/8ocX8+yv8ftu8ebPFsaSomGZyREREJEp6yBEREZEo6SFHREREolTwNTncBRfwO+Hyuo1wHQSvq+DcPuCXKnM+Pyxbbtu2bbk/P8xBct6ZOyOH5cK8g+7cuXMt5vUhgL9GJ7b1HPzeA8Dll19ucdeuXS3O1tU425oX3lGcyxcBvySZ1+GE6zP23ntvi/v377/T55TtmqmTsY/vF2DHVhDf4q7GQO6djTt06GDxRRddZHG4/o6vyymnnGLxkiVLcvp3pHy8W/Vhhx1mcfjdyO//xx9/XPwTk3Jl6zrfpUsXi7mlB7dhAYAxY8ZYvHr16kKfYuo0kyMiIiJR0kOOiIiIRKng6Sou8Qb8lAZPmXHJOOBPfzZo0MA7xlPVnFrI1o02WwqCyx85JcWdcwG/lP3uu++2eObMmd648L8lJmHn4nr16mU8xnJNB/H0KHfFBYAPP/yw3J/RvHlzb9xBBx1k8X777WdxIboVf/rpp97r6rjhYO3atS3m1BDgX2cuGx8+fHhOP5s7JgPA//7v/1o8ePBgi7kTL+B/Hs4991yLw41/JbvwHhkyZIjF4dIDxt+b1fGeqCr492337t29YzVq1Cj374TtHXiDY76usdBMjoiIiERJDzkiIiISpYKnq8KUzx//+EeLr7zySouXLVvmjZs9e7bFvXr18o5xtVW2TrrcrZF/Pnd0BICJEydavHLlynL/PPwZvHlnbBVUIZ7CDqdAGzZsaDFXxO1M12DuUDty5MhyY8DvnsrTsuedd543jjdPDdMfmeR6fmPHjvWOcXqtukzTX3jhhRZzOhDwKzW4oirsSs14M94HH3zQO8abQvJ1eOGFF7xxffv2tfiMM86w+Pbbb/fGVZdrlK9ws0buUB8eY/w9v2bNmsKfmOSEK0jDZR78Pc5pqPB79qWXXrI4xqUXmskRERGRKOkhR0RERKKkhxwRERGJUsHX5IS7mP7jH/+w+KmnnrI47CTLayS+973veceOPfZYizlPvHjxYm8c7yL+5JNPlvuzAeXpvwvncsM1GJz3zbdEe968eRaPGjXK4nCdVo8ePSzm0mUuGQb8rsy8NijXndDDXaz5/ML8dXXY8TrsFD1gwACLw/eUS75///vfZ/yZ/PcuvfRSi3kNDgCsX7/e4gsuuMDil19+2Rv31ltvWVyIVgHVFbf4AIA2bdqUOy78zuT7QjuPV55wnRR/R4Zrcnh9zaZNmyyeMmWKN66Y1y/XViLFpG8HERERiZIeckRERCRKBU9XZZMtTcSlv5dddpl3jKfouGT1kUce8cbxFGqMnRvTEKYCsqWAMglTkwsXLrSYN/fj0mLALw3nONw0NFvnZZZpujRsMTB//nyLY9yw7rtwehjwOw+HU9t8D/LmuSFuC3HVVVdZHKb/uM3E6NGjLa5Vq5Y3jru5hilyyY43uD355JMzHmNhKnnSpEmFPzH5TrwJNQCcddZZFvPG00DmliphZ/lCb0LMvzO6detm8YoVK7xx/L1bzI2QNZMjIiIiUdJDjoiIiESpUtNV3j8cbLh36qmnWsypCcCfduNqnGHDhnnjsk2XS+44FbDnnntW+OeFaSLe9PInP/mJxWEl18CBAy2uU6dOTv9Wrqv5+TM1btw47xh/xrjTdXXBXW8Bf/r5gw8+8I6FaYxvhZ2nn3vuOYs59cTXHwAef/zxcn9eWAW09957W8xdjlU5+d06d+5scZiuYpxKfPTRR71jEyZMKPyJSbn4d+WJJ57oHePvyHBpwYIFCyweM2aMxbzxcTFkSld98skn3jj+PaB0lYiIiMhO0kOOiIiIREkPOSIiIhKl1NbkhDn7/v37WxyWMU6ePNniX//61xYvWbKkKOdWHXGZfvPmzS3m6wLsuJZqZ3824K+54vU/YSl4pk62uXYyDnEHUF5bcvfdd3vjuIQ8xl15K+LZZ5/1XmdaAxPuFN+iRQuL33vvPYu5MzngX79mzZpZ/Ktf/cobx5+BESNGfNdpV2sdOnTwXt90000Wd+/e3TvG7/8bb7xh8fjx471xrVu3tvjwww+3OCzn5w7KfD/zegzA37X+nXfesVjf8f5nPexqzN3Jw+8q3l187NixFhe7azuvr5k9e7bF3C4kHFdMmskRERGRKOkhR0RERKJUqekqnq685pprvGOHHHKIxR999JF3jLugrly5skhnV73xNDVPJc+aNcsbxxvChRs55irXsvRMaalcU1JLly71jr3//vsW//3vf7d4zpw53rjq2EGX702+xqFsU8ycejzjjDO8Y/ye3nDDDRaHHZT5Z3Dnc+66DPjXtjqW+e+MsEycy47DVDLjVgJ8vwD+PcipyDBlwq0f+H4O05zc/ZbbDfzsZz/zxn3xxRcZzzcmfD9yiipML/K48HuLOwxX5j3C3xH8+0MbdIqIiIgUkB5yREREJEqVmq7i9MYPfvAD7xhvLvb22297x3jKSx1Ni4OnGLkz5auvvuqN47QBpxYKsZFnNrlOdW7cuNFirsoD/P+WadOmWVxdpsCz4esXVj7mirsQc/UNACxfvtxi7jAdfk7OPPNMi6+99lqLw6n40047zeL169fndb4x4wrVsEIy3Ow0k4YNG5YbF0JYpcnne9xxx1kcpqu483K4rCEmfD9yyi+8N3kcb14N+JVp4bHKklaKimkmR0RERKKkhxwRERGJkh5yREREJEqVuianS5cuFnP+HvA7qf7iF7/wjmkdTuXi9Q8TJ070js2cOdPifffd12Lukrwzct01nI+F5ZCbNm2ymHP2f/nLXzKOq45l4tnwPRZ2Ne7Tp4/F9erV847x9ePWA+FOx7xG5ze/+U3G87joooss5jUXvD4H8LsmSxle58LvI69xAfxrFq6JynQPhq0DuFSc23p8+eWXO3HG/8Fl0hyHu24/88wzFse8Jofxdc3WfiN8P3gtK3/3VTeayREREZEo6SFHREREolT0dBWXuLVv397iMAXFHUxVElp1hB2muft07969Lb7kkku8cW3btrW4bt263jEuPc+Gp77nzZtncbihJqfQuGxSnXDzk21qmzdXBYD777/f4mXLlll8xx13eOMeeOABi3/+859n/Pn8vcCpCS47B6pGaWpVxmnFndnQljdv5JTjjBkzvHHcPZw3fww3Ycwk7LTcrVs3i7mr76hRo7xx1b3jfZg25LT71KlTvWOcvkpro2FOr4XdzStrGYpmckRERCRKesgRERGRKOkhR0RERKJUqWtyOnXqZPH8+fO9cY899pjFabWglh2FJaFcusu7d4dbKPTq1cvisBV5WIacCe9MPGHCBIvnzp3rjeNtGbRWo+L4XgT89VYdO3b0jr3++usW867H4bqrcNuPb61du9Z7fdVVV1k8YsQIi9NaU1Cqsu0Wz+9luG6N11iNGTPGYt6WA/DXbeVbNs749wH/u9W1fQhfP36vJ02a5I3jNUrhWkW+tpX5vcjrrbj9BP++APy1t8W8vzWTIyIiIlHSQ46IiIhEyWWbxnLOVXiOi0vIuPw0TG9wCXl1TzkkSVLYLby3K8T1zCTcVZh3LeZddMsbmwlPVa9evdpiLnMtBcW4nsW8lqEjjzzS4rCLdJs2bcr9O+E9zOmOkSNHWszpEQBYtGhR3udZGUrl3uQU8dlnn+0d4064U6ZM8Y6F6cPYlcK9yS0AGjdu7B3j79awy3gh0ogVVZkl5JmupWZyREREJEp6yBEREZEoFT1dxXjVtaolMiuVKXHJTSlMieeqSZMm3mveTDEbrgIJp61LSSnem2F6mCt3slVhVQcx3ZvVndJVIiIiUq3oIUdERESipIccERERiVKlrsmR3JRi3l8yU94/Hro346J7Mx5akyMiIiLVih5yREREJEpZ01UiIiIipUozOSIiIhIlPeSIiIhIlPSQIyIiIlGK6iHHObeHc+4h59xS59wnzrkZzrlj0j4vyZ9z7lHn3Crn3Gbn3Dzn3E/TPifJn3NuiXNuYNrnIRXnnHvVObfRObdH2uciFeOcO9M595Zz7tPt37fjnHOHpX1ehRDVQw6A3QAsB3A4gPoAhgJ40jnXLsVzkor5DYB2SZLUA3ACgFudcwekfE4i1dr279R+ABKU3ZdSopxzvwBwN4DbATQH0AbAnwGcmOZ5Fcpu3z2kdCRJsgXAjfRHY51ziwEcAGBJGuckFZMkySx+uf1/HQG8nc4ZiQiAcwFMA/AGgPMAPJXu6Ug+nHP1AdwM4EdJkjxNh8Zs/1/Ji20mx+Ocaw6gM4BZ3zVWqi7n3J+dc58BmANgFYDnUj4lkeruXACPbf/fUdu/a6X09AVQE8DItE+kWKJ9yHHO1UDZDTgsSZI5aZ+P5C9JkosB7Imy6fGnAWxL94xEqq/tazXaAngySZK3ASwEcGa6ZyV5agxgfZIkX6V9IsUS5UOOc24XAMMBfAHg0pRPRwogSZKvkyR5DUArABelfT4i1dh5AF5MkmT99tePb/8zKT0bADRxzkW1dIVF9x/mnHMAHkLZAqpjkyT5MuVTksLaDWVrckSkkjnnagH4bwC7OudWb//jPQA0cM71SpLk3fTOTvIwFWUz4ycB+GfK51IUMc7k3A+gK4DjkyTZmvbJSP6cc82cc6c75+o653Z1zh0F4AwAE9I+N5Fq6iQAXwPoBqD39v91BTAZZet0pIQkSfIxgOsB3OecO8k5V9s5V8M5d4xz7ndpn18hRLV3lXOuLcqqqLYB4BzjhUmSPJbKSUnenHNNUfb/XfRC2QP5UgB/SpLkwVRPTPLmnFsC4KdJkryU9rnIznPOPQ9gVpIkVwV//t8A/gSgVczrO2LlnDsLwJUoe2D9BGXVq7clSfJ6qidWAFE95IiIiIh8K8Z0lYiIiIgeckRERCROesgRERGRKOkhR0RERKKkhxwRERGJUtZmgM45lV6lIEkSV4yfq+uZjmJcT13LdOjejIvuzXhkupaayREREZEo6SFHREREohTd3lUiIlJadtvtP7+Kvvnmm3JjkXxoJkdERESipIccERERiZIeckRERCRKJb0mp0aNGt7rvffe22LO8Wbz1Vf/2TB3zZo13rFt27ZZrNxwceyyyy7lxoXA11aqDr7OzmWu4OXNg3X/xaVZs2be6wsuuMDi9957z+IXX3zRG8ffyVL1ZbvXK+v+1kyOiIiIREkPOSIiIhKlkktXNWzY0OKePXt6x2644QaLOXWVzSeffGLxyy+/7B0bN26cxZMmTbL466+/zu1kq5E99tjDe73XXntZHKYO69evb3GbNm0s3m+//bxxuaaveKrz008/tXjkyJHeuNWrV1v8+eef5/SzpfAaNGhg8Z577plxHN+bHAPAl19+WfgTk6Jq166dxT/5yU+8Y1dffbXFs2fPtnjx4sXeuJkzZxbn5GSn8Hdz+N3P9/Shhx5qcfg7edasWRZPnDjRO1bI9JVmckRERCRKesgRERGRKJVEuuq4446z+MQTT7T4lFNO8cZxKouF6SVe1c3Tbn369PHGnXrqqRZff/31Fj/++OMZf151wu/doEGDvGO//vWvLa5Xr553jKc369SpYzGnMYDslTeM3/8vvvjC4vB6PvfccxY//fTTFqtio/h23XVXi3v16mVxt27dMv4dTlu8//773rGPPvrIYlVeVV21a9e2+LzzzrP48ssv98bxd0KrVq0s3meffbxx/JnQdS+8bNVQbdu2tZiXFvTr188b16FDB4s5XRX+Hp48ebLFxby/NZMjIiIiUdJDjoiIiERJDzkiIiISpSq5Jmf33Xf3Xv/4xz+2+Nhjj7U47HjMebwlS5ZY/Oqrr3rjuBy1f//+Foe5RS55vO666yx+9tlnvXGbNm0K/xOqBc7ZduzY0TvWpUsXi2vVqpXx7xW64zGXqx900EHeMS47Hj9+vMVak1N4YT6f11wcfPDBFh9xxBEZfwav5Vq3bp13jFsFqB1A1RHew0ceeaTF55xzjsV169bN+DP4GH8Hhz9fa3Lyw+vjAP/3KJd/h7+H+ffjgAEDLObfoeHPaNSokcUff/yxN65p06YWh2Xoua7HzIVmckRERCRKesgRERGRKFXJdFWY3uDUB5cLL1q0yBv36KOPWjxs2DCLw403OW3xxhtvWMzlzADwve99z+LGjRtbHKbJqiu+FgsXLvSOzZ071+KwhDyftBS/5+HP4ylR/tncdRnwyx55Snz9+vU7fT6yI55iDqefW7ZsafH3v/99i7t27Zrx53EaeM6cOd4xvqeVrqo6wuvOqQzubp4Np495CQJQfdt15IPTUnxdwnYfvHNA7969LQ5TinyMu9aHeDnIu+++a/EzzzzjjXvnnXcsDr+DC7mrgGZyREREJEp6yBEREZEoVcl0VbgZH6ehuDLjzjvv9Mbx5m0bNmzI6d/iKbMPP/zQO6ap0ey4uiHcOG/EiBEWh9Oe+aSrOF3Yo0cP7xiv+uefvWXLFm8cV9yFxyQ/PCXO16hv377euJNOOsnio446yuIwvfHVV19ZzOnF7t27e+O4Q2pYeSWVi9OU4XU6+eSTLebKx7B6hr9redlAmOIqZNVNbML3hu9HTt3zNQH8DuS8iWa4LCPTZrpLly71XvPvAu5q/OKLL3rjeMNk7lRfaJrJERERkSjpIUdERESipIccERERiVKVXJMTdrL8zW9+YzGvAci3zIzXAVx66aUWH3bYYd44Xt+xceNGi7kEXcrwehcAuOeeewr68zt16mTxGWec4R3jnW7ZlClTvNdPPvmkxWFpquSH8/bNmjWzmMtSAT/vz/dfuI6A1+R89tlnFodrqHicpIt3pw53F+c1Htnw5yBTR3TZUZMmTSwO3+srr7zSYi7/3nfffb1x3NmYf/eG62RmzZpl8fz58y0ePny4N47Xyy1btszitO5ZfYJEREQkSnrIERERkShVyXRVNvmkqBo2bOi95ql0nl4NOx6vXbvWYp6SCzcakx0VemqSf162jfn42OLFi71j/LqQHTWrkzC9VLNmTYtbtGhhcbixIm/Gxz8jvJa88SanQMN0qFoApIvbQpx33nkWc6sAYMcWAd8K23OoNDw34aaZnIYKNyTm1hp8b4bXhK8Ft17hEm8AGDlypMWckpo6dao3jn8/VoW0smZyREREJEp6yBEREZEo6SFHREREolRya3JyxaV1gwcP9o5dccUVFnOOM9zVnNfhPPjggxZru4fSEOb5M5Wp6noWRj6lv+F7z2tteB0Ol6IC/todKT7+PgWACy+80GJekxOua+Try9v11K5d2xsXbiFQ3WVa63baaad544455hiLeRsUwF8Hx2vfeK0pAKxatcriu+66y+IZM2Z44+bNm2cxl5dX9fWNmskRERGRKOkhR0RERKIUVbqKpzwvvvhii3k6FQBatWpl8b///W+Lhw4d6o3LZ1dzKQ4uRdy8ebN3jDtQ8zTviSee6I1r1KiRxddee63F4fStZBaml7Zt22Yxd5EO75dMXcLDn8fjOHUVpqeqQmlq7HjX8AEDBnjHLrnkEos5nRJe5w8++MDi8ePHW3z66ad74/g7mYU7X/N3fEyfgbCs+9RTT7X46KOPtjgs0efvu/Be4pYZXPL96KOPeuM4DTV37lyLw47HpZrW10yOiIiIREkPOSIiIhKlkktX8fTcoEGDvGNHHnmkxRdddJHFvLkmAPz5z3+2mDf/VNqi6spUdQP4VRtcLde8eXNvHHcH5Sl2XffchdPqe+21l8U9evSweJ999vHGcYdcFqY3+Fpu2rTJYk6LAaU7dV7V8QbIhx9+uMVcTQX4m7HytZg9e7Y37rLLLrO4QYMGFp955pkZz4HTZHwOgH9Ph98DpYbf6/r163vHBg4caPHBBx9scViVxr/bwk78kyZNsvjVV18t988B/54L77MYaCZHREREoqSHHBEREYmSHnJEREQkSiWxJofzv9zh8brrrvPGtW3b1uI5c+ZYPGLECG/cQw89ZLHWY5QGLk+eMmWKd2zcuHEW88677du398Z17tzZYi4v5xJKAPj8888rdrKR4e7FYUlv9+7dLeb1E/zn4d/j7qthaXimncfDXcez7UQvuQu7gvM9c+mll1ocro3hzwSXKv/hD3/wxr355psW8/czrwMBMq+xqlevnvea1+uUOm5pceihh3rH+Pdc48aNLV63bp037oEHHrD4lVde8Y7NmjXL4tjX3WSjmRwRERGJkh5yREREJEpVcu4vnJK89dZbLT7hhBMsDjeN++yzzyzmjpxvv/22N27r1q0FOU+pPLwJXFgqOXHiRIt5Crhdu3beOC4v52PclgDwp3NVquynJsL0AacAeYPAsHyf33tONYVpqEybcoZpLaWrCqNXr17e67///e8W9+zZ0+IwrbV06VKLb7/9dotfeuklbxx3zV2+fLnFY8eO9cb94he/KPf8wo07w1LrUsabmYbfVZze5c7O06ZN88a99tprFnN3acBP8Vfn+0UzOSIiIhIlPeSIiIhIlKpMuirbZnC8KVnDhg0tDjteDh8+3GJe1V/dVpPHLpx65alzTnFkm6Jdv369xWE1lVJUPk4ZcNdawO8czfdm2BmZ0x18XcJNALkKhOOYNmNMG1fr3HDDDd6xfffd12K+ZuE9wZ12+fv5kEMO8cbdf//9FnPKkdOX2YRpz/79+1v87rvvesdK7TPCv/PCNDCniFetWmUxL90AgHfeecdiTunLf2gmR0RERKKkhxwRERGJkh5yREREJEqprckJS+auv/56i4877jjvGOfwOSfJ3R4Bv3ux1lXEJduOvdyNtVu3bhaHZa+8/oM/K6WWy69svC4i7Hx7xBFHWMw7kodtIPha8O7iM2fO9Ma99957FvO6D603yF94H/B3L3fWBTKvlQl/BpeXcxzuKs+fD77PuNVDNmEJebh2JRbh+8u/v/h9CztF6774bprJERERkSjpIUdERESiVKnpKp56HDhwoHfs2GOPtTjcBPC3v/2txVySqM0148afF05Dfe973/PGXXjhhRZzeWxYnjx//nyLx48fb7HSVdlx6qlu3breMe7amm3zRJ5+5y7HYRuITF2Oq3PH1orijTEB4IorrrA411LuEJc4s7B1QJs2bcodl+16hqmb6iDb+8H3Vfi7MVuZv5TRTI6IiIhESQ85IiIiEqWip6t4+vLAAw+0+Fe/+pU3rnbt2hb/85//9I7dd999FnOnWqm6cp1G5aqppk2besc6duxoMacsO3Xq5I3jv8fVBtOnT/fGPffccxbPmzcv4zlJcXF6MNygk1NUSiPmj79PzzvvPO8YdygO780NGzZYzNVtuQo3Ul23bp3FmzdvtnjNmjXeuIsuushi/k6IGW80/P7773vH+L3i6sbLL7/cG3fTTTdZ/OGHH3rHeMPq6kwzOSIiIhIlPeSIiIhIlPSQIyIiIlEq+JqcsIz06KOPtvjGG2+0uH379t64lStXWvy3v/3NO1bV1+FwOWVYQtmyZUuLP/roI4vzyXdXZWEpKq8JCHf55vU6XBJ52GGHeeN69epl8T777GNx2C2Vyy/5fX3ppZe8cf/+978tDsvLJTNeGxOuueAOrPyeZut4LMXB3z0nn3yyxeeee643ju9N7jANAHfffbfF06ZNszjXzrrhOqpt27ZZzN2QmzVr5o3jNhDZ1uTw90XYDbnU1nDxvbNw4ULvGO88zm0a+vbt643j70zekRzw1x3ye19q71NFaSZHREREoqSHHBEREYlSQdJVTZo0sfjiiy/2jvFrnqIMp695c7+hQ4d6x7jL6oQJEyyuzBK5mjVrWhxuLjpkyBCLzzrrLO8YTzVeeumlFo8ePdobV4odXTlNd+SRR3rHTj31VIvD1Bxfe05bHnrood44nprmdFiY/uK01FtvvWXx+njqgQAAIABJREFUgw8+6I3jdGEpvt9p4RTVggULvGOzZs2ymNOIe++9tzeOUymcyuL7I3zN48KNH2VH/J5zqXH4fcUppMsuu8w79uabb5Y7rtAaNGiQ07gw7ckbxHJpNbBj9+yqjr/H5s6d6x3jtCGnHgcPHuyNu+OOOyxevXq1d+zFF1+0mEv2X3nlFW8c39+F3vCTf0eHSxr42oYpNC6h5+/tfGgmR0RERKKkhxwRERGJUt7pKk45HHXUURaH3TU5RcVTzjwdBfhT3TwlCfgbzD366KMWP/3009443oCR/60wNcZTpfXr10cmfOy4446z+IwzzvDGcbounIbl6oXYKnr4feUqMgAYMGBAxr/HaS5OT4RVU/zzucKOKw8AYNSoURbPmDHD4jBNVt2qCgqFu+ByuhjwN9TkdMGJJ57ojeMu1fXq1bO4d+/e3rg5c+ZYzFPs4VR8oafVS1GYyuG0Rvfu3S0Ouxq/+uqrFk+dOtU7Vlnva3gv8vd1to1e+bOTbVypCX838PcYpxvDZQENGza0ONy8k5d5bNq0yWJeGgL46ap8NvkM/w6/5u8H/j0J+FV+/B0D+B2gX3vtNYvzWWagmRwRERGJkh5yREREJEp6yBEREZEo5Z3U5HUy11xzjcWtWrXyxnHudfbs2Rbfe++93jgur+7SpYt3rEOHDuX+W7ybLuCXEnM3SV4DAvj56h49eiAT7qjZokULi8OuxlxqGe4me8EFF1i8aNEii2MoYeb8/RtvvOEd4/c/3DWcc7H8PoR5WV57wyWVnK8G/HUcnNvOJ78sO+L3MSwr5mvEHVfDEvJMJeC8ji78eVxiq2u5I16PAfjdcLlcd/ny5d64F154weK0vofCXcj5uvP3fXURfr75O23YsGEWhx3HDznkEIvDVgGtW7e2mH9/devWLeu//a1wLWv4ezTTOMa/B8IO1dzZ+plnnvGOrVixIuPP3FmayREREZEo6SFHREREopRzuircNG3QoEEWd+3a1eKwO+GIESMs5vLT8ePHe+O40yaXQgLARRddZDGXGYeppnAaLhOedss0BQf4qTae0gtTUiNHjiw3DsfGPOW+du1a7zWnlMKutlzCuGXLFosnT57sjeP3jo+F5cTF7MwqvrDE+OOPP7aY2yWEn/UPPvig3J8Xpqu4gzJPzceQ3i20sP0Ffw9zCiHspvuvf/3L4rS+k8J7dvr06RbzUohsm3XGjNPunMoLf7/wEoiOHTt6xzh9ye9pWGrO+HMTlug3btzYYv4e4N0AAP+acZqavyvCnx9+PyxevNjiit77mskRERGRKOkhR0RERKKkhxwRERGJksuWk3XOJRR7xw444ACLuVSNy0gBYOXKlRbnu5Nw06ZNy/23wu0fuOU3r9fhkvHQs88+W+65AsCkSZMs5nxiWN4W7opdUUmSZK7JqwC+nkX42d5rzgGHa6c4d7x06VKLp0yZ4o3jrT8K/R5XpmJcz2Jey53B153LlrO1mGdhSSyXnFbFdgBV6d4M2+TfdtttFh922GEW8/cYAAwdOtTisG1DWk499VSL//SnP1kc/jfyWr8hQ4Z4x/LZhbzU781s60u5xQCvi8y2HQavpwnv2QMPPNDirVu3WsxbNwH+Gh1eaxNeHz6/efPmece4xUCuay4zXUvN5IiIiEiU9JAjIiIiUco5XRXKVHpdmaWe2abdeCos207j2dJpae1aXZWmxAsh/KxwioM/f7GWCZf6lHg+wmue6/dFVf8MVKV7M0wRc4lvgwYNLOYdqAE/RVVV0oCcljrjjDMs5v8OwE9Xvfjii96xfFpJVMd7M1+ZyvmzdTzm+zm8tzP9HsiX0lUiIiJSreghR0RERKKUd7pKiqcqTYlLxWlKPB66N4uP0yJhKiRb+iMfujfjoXSViIiIVCt6yBEREZEo6SFHREREopTzLuQiIiLFFu50L1IRmskRERGRKOkhR0RERKKUtYRcREREpFRpJkdERESipIccERERiZIeckRERCRKUT7kOOdOd8594Jzb4pxb6Jzrl/Y5yc5zzi1xzm11zn3qnFvjnHvYOVc37fOS/DjnznTOvbX9eq5yzo1zzh2W9nlJfrbfnwPTPg+pGOfcHs65h5xzS51znzjnZjjnjkn7vAoluocc59wgAHcA+BGAPQH0B7Ao1ZOSijg+SZK6APoAOBDA0JTPR/LgnPsFgLsB3A6gOYA2AP4M4MQ0z0tEsBuA5QAOB1AfZd+xTzrn2qV4TgUTXXWVc+51AA8lSfJQ2uciFeOcWwLgp0mSvLT99e8BdE2S5LhUT0x2inOuPoCVAH6UJMlTaZ+PVJxzbjiAswBsA/A1gJuTJPldumclheKcew/ATUmS/Cvtc6moqGZynHO7ouz/22/qnFvgnFvhnLvXOVcr7XOTinHOtQZwLIB30j4X2Wl9AdQEMDLtE5HCSJLkHADLsH2mVQ848XDONQfQGcCstM+lEKJ6yEHZNHgNAKcB6AegN4D9oRRHKRvlnNsE4DUAE1GW7pDS0hjA+iRJvkr7REQkM+dcDQCPARiWJMmctM+nEGJ7yNm6/f/ekyTJqiRJ1gO4E2UzAFKaTkqSpEGSJG2TJLk4SZKt3/1XpIrZAKCJc0575YlUUc65XQAMB/AFgEtTPp2CieohJ0mSjQBWAOCFRnEtOhIpPVNRtnbjpLRPRApK362RcM45AA+hLBtyapIkX6Z8SgUT1UPOdn8HcJlzrplzriGAKwGMTfmcRKqtJEk+BnA9gPuccyc552o752o4545xzmktR+laA6BD2ichBXE/gK4oW2MV1Wx5jA85twB4E8A8AB+gbKHqbamekUg1lyTJ/wL4BcrWx61DWcnqpQBGpXleUiG/ATDUObfJOXd12icj+XHOtQVwIcrWsK7e3sfqU+fcWSmfWkFEV0IuIiIiAsQ5kyMiIiKihxwRERGJkx5yREREJEp6yBEREZEo6SFHREREopS1A6lzTqVXKUiSxBXj5+p6pqMY11PXMh26N+OiezMema6lZnJEREQkSiW9l8wuu+yS8fVXX2kvwFjttpv/sW3YsKHFderUsXjTpk3euPC1iIjETTM5IiIiEiU95IiIiEiU9JAjIiIiUSq5NTlNmjSx+Lbb/H03e/fubfGtt95q8ZgxY4p/YlJwzZo1s7hfv34W//KXv/TGtWnTxuK6deta/NFHH3nj/vznP1v8u9/9Z/Prr7/+uuInK9h1110zHuM98r755pvKOB0pIuf+U8jSqFGjjOP4HtQ+iaUlvJ/5vuXrH66NzSStdbKayREREZEo6SFHREREolRy6ap27dpZPGjQIO9Yq1atLO7Tp4/FSleVho4dO3qvx40bZ3Hbtm0tDqc9edySJUssPvvss71x119/vcXLli2z+LHHHsvvhKshnqYGgN13393ivfbaK+Pf+/TTTy3esGGDxUphlKbGjRtbfMstt1gcfj74nlu7dm3xT0y+U7YWHK1bt7b4gAMO8MbNnz/fYl420rVrV28cp682b95s8YQJE7xxq1evtnjdunXesUJ+L2gmR0RERKKkhxwRERGJUkmkq/bYYw+LL7/8covD6XFVcKQnnKauX7++xWGnYZ7O7NGjh8VPPfWUN27p0qUW/8///I/Fb731ljdu+fLlFnOl1KOPPuqNmzx5ssU333yzxePHj/fGaVrdx1UWnKYAgL59+1r8q1/9yuLw8zBr1iyL+b3/8MMPvXFffPFFxU5WKkWDBg0sHjBgQLl/DgAvvPCCxaNHj7ZY38/FV7t2bYtbtGhh8cknn+yN43uYK5Q5JQUAn3zyicX8O5m/6wH/3v/yyy8tPvfcc71xM2bMsPjuu+/2js2ZM8fibdu2oSI0kyMiIiJR0kOOiIiIREkPOSIiIhKlkliT0717d4uHDBlicVgK9+6771rM+V8pPi7xBoCXX37Z4iuvvNI7xm0Abr/9dovDtTE//elPLV6/fv1OnxN/HgDgsssus/jBBx+0+N577/XGnX766RZr7YCffw/XwXEOv1u3bhl/Bq8P4Ou/ceNGbxzn8FVeXnVxG4ctW7ZY3L59e2/cfvvtZzG38tB9VXg1a9b0Xu+zzz4W833Kv0MB/7uby8nDjsd16tSxOFvHY7623GKiU6dO3rg999zTYm79AQDDhg3LeGxnaSZHREREoqSHHBEREYlSlUxXhWmoI444otxj4ZQnl6HNnTu3SGcn5VmzZk3GYw899JD3mtMQr7/+usVXXHGFNy6fFFWmfwcAXn31VYu5VPnAAw/0xnF6ZuvWrRU6hxhw+SlPewPA/vvvbzG/byoFjxvf73xfhSlLLi+uUaOGxWlt1hibevXqWXz++ed7x3784x9bzPdw06ZNvXH8Pcnl2itWrPDGcSqZhb+HP/jgg3LHHXbYYd5r3qHgqquuynhOvJny559/Xu7PzkYzOSIiIhIlPeSIiIhIlKpkuiqc8jznnHPKHRemM5577jmLK9olUXZOOJXJrxs1auQdW7x4scUXXHCBxRVdRf9dePqVp1R79uzpjeMN56ZPn17Uc6qKwnQxd0jlyjMA6NChg8W8USp3MwWAd955x+Jp06ZZHN6nqqgqDXx/cyfcsCJn4MCBFj/yyCMWv/fee0U8u+qDK57CDav53uS0O2+0CQAzZ860eOrUqRZzGhIAPv7445zOicfVqlXL4ptuuskbx5+Nli1bese4izYvd1i5cmVO58A0kyMiIiJR0kOOiIiIREkPOSIiIhKlKrkm58gjj/Rec+dG3mWad5UGgI8++qi4JyYZhWspeH1Nx44dvWPXXnttueOKjctWf/vb31o8YsQIb9zgwYMtro5rcrjUF/DLT8Mdh3lNzbPPPmvxm2++6Y3j3cb572gNTunbvHmzxWFpOH938/f67NmzvXEqKc8Pr58LS7d5Tc2iRYssnjVrljeOu45zXIhrwusdud0EADRu3NjicG3exIkTLd6wYUOFzkEzOSIiIhIlPeSIiIhIlKpMuopLDw899FDvGHdSXbZsmcWccgD8VJZUrnCDzl69elkcdsQMy4vTwGXs2izQv/944zzAL0WtW7eud+yzzz6zmMvEFyxY4I3jElalqEofpzImTZpkcdj5nLvacndeKYxPP/3UYk4XA8Dy5cst5jL/TZs2eeP492Yhvgv5u4TTlZz2Bvy0eLjUhFNqmTot50ozOSIiIhIlPeSIiIhIlPSQIyIiIlGqMmtyeE3HMccc4x3bZZf/PIu99dZbFvP6HElXuPVGs2bNLK6K5aHHHXdcxmNhbjtWnDvncs7+/ft743j34LCEPFOuP9wtWOvl4sVt/LOtn+DvcedcUc+puuCSb14bBaS39q1169YWn3322Rbzdwzgfx54/RDgt+6o6HeHZnJEREQkSnrIERERkSillq7iqSrALznmknHAn6564oknLM61w3H4b6lkuDBq1qxpMe9U/V0aNGhQjNPZQTglzt03zz33XIvDDqBhN9ZYcQknl3fut99+3jguKQ/fU5WDV33hNeO0QdgSINyBPhft27e3OOyWzd+93bt3tzj8DuCutvp+zk9a92L4mTnwwAMt7tu3r8Xh72FObb788svesVWrVhXs/DSTIyIiIlHSQ46IiIhEKbV0VZiS4i7H4bQWb+L4yiuvWByu1j7rrLMs3nvvvS3m1d4A8NRTT1nMm5iFG4FpKj47roLjqWjAnyIPpzMffPBBi/m6c/fcfPHn6vTTT/eO3XvvvRavXr3a4tNOO80bF1YGxYrTULyJangtd999d4u/+OIL7xh3XK2KVXTVFaeNunXr5h274oorLA6vdVg9t7P/Fn/vAv53OW/Q2Lt3b28cV9fw93D4naxUVtXA37NHHXWUd4wrbbnLdbgZ83vvvWfxuHHjvGPhhp0VoZkcERERiZIeckRERCRKesgRERGRKKW2Jod3JwWAgQMHZhzLeb3rrrvO4nDna84Ncs4wLKEcMGCAxaNGjSr3ZwPA+vXrM56T+Hl17p4LZF/PxNft8ccft3jYsGHeuPfffz+n8+jRo4fF559/vsXHHnusN47z+7yeaOHChTn9O7HhtWqHHHKIxX369PHG8bUMd5meOHFiuce0PiddvDbmzjvv9I59//vftzgs+ebuxeFu1YzX2TVv3jzjz+PvXt6R/C9/+Ys3jtd6vfvuuxY/8MAD3jj+vGl9TuXidgOdOnWy+MYbb/TGcQuKpUuXZhz3xhtvWLx48WLvWCGvrWZyREREJEp6yBEREZEoVWq6issJhw4d6h0LSxlZo0aNLL7yyitz+reybQDXtGlTiwcNGmRxOIWaa7rq8MMPtzjcTOy1117L6WeUCp6mHjx4sMVheeAHH3xgMb/HgF/q/8c//tFiTl0BmacswxYDfE6Zpr0B4Oabb7Z4wYIF5f7smIXvG6cNO3ToYDGXlgN+uiosIecNOpWiqjr4ngi7C69du9bicEPNmTNnWsydwMN7kT8jfH937tzZG8fLBjiV1bJlS28cf2/WqlXLYm7xAey4CaUUT9jmhVNU3NU4vJb8mZo2bVq5MQB8+OGHFhdzA1/N5IiIiEiU9JAjIiIiUarUdBVPQ4YVHFydky3VlK1qJ9N0eThNz6+5MoDTTgAwffr0cn9eOI13ySWXWPzOO+94x2JLV/EKe37vnnzySW8cpzHCdBVPl3NVT64bd3I1FQC0a9fO4tGjR1u8bNkyb1x1T6eE9wG/j9k2T+SKm3AzU05vFHPKWXYOb178pz/9yTvGVYvcsRoANm7caDFXV4Xfu5x6Gj58uMVXX321N467ifPfmTp1asZzWrduncXhZrnqQp9Ztt9zueLfw0cffbR37IYbbrCYU1ThzgOcYrztttssXrRokTeusq6lZnJEREQkSnrIERERkSjpIUdERESiVPQ1OZzj412g27dvn/HvhLk6fs252/Hjx3vjRowYYTHnmsM1HKeccorFp556qsXcWTmbsMT24IMPtpjXKMRoy5YtFvOu4Zs3b874d8JdyLkT8S233GJxriX71bH8uxh47Rvn78M1cbwre7grdPhaqgZek/Poo496xwqxdorXt/Fu0n/4wx+8cT179rR43333tTgsBf/HP/5hMd/fYel6dexyzL9DuZ0KADRs2NDisA0Ldx7OdX0OjzvssMO8Y/x7lK/DnDlzvHFjx461mMvE01pPpZkcERERiZIeckRERCRKBU9XhVPdPGU2ZMiQjONYmLZ49tlnLeZybU6XZDN37lzvNae8TjzxxJx+BuOpYMAvm4y9jLYQU475lDZK4fG15Onn8Bpz64cWLVp4x7h8nzfc27Ztmzcu9vuiKiv2e8+fl3nz5nnHeElBuClzpp9RHVs9hG1JunTpYjG/b/w7FPDTgZy6AvxWENl+32YSfk/zz+DfvWEnY970mluJpEW/bURERCRKesgRERGRKBU8XRV2P7ziiissrlmzpsXh9BlPV/JKewD4zW9+Y3GuKap85LpyPxzH03PhtGPMqmOlQykLrxenbTkOp715Q9uBAwd6x7iigzf8DKsMFy9ebDF3UA5xqmLNmjXl/jngV+zVqVPH4jDVxpsFZqsAlMIIN/zkdAV/54eVQN26dbN4/vz5Fsf8HdOkSROLebNjwP+9uddee1kc/n7lyqtCyJa25vQVd74//vjjvXGrV6+2+IknnrA4XDYSflaKRTM5IiIiEiU95IiIiEiU9JAjIiIiUSr4mpxwB2PeZZplK0XmsnPA77z41ltvWcw5eyBzji/cXfzSSy+1mDsycqn6zvjiiy/KjWPEayP4/QrLDWPOpZeq8JosXLjQYl5DE5b68pqccM0Z70Z88sknWxx2GV+yZInFvE4jPCc+xl1xudM24K/v4zL28DuAvyOmT5/uHdNntHLx+pFs3XnHjBljcWzXiL8n+R4JW5l06tTJYv6sZ1uDk61MPNvvW24xwO1RwnuuefPmFvOauHCngBNOOMFiXkO7du1abxzvcl/M35uayREREZEo6SFHREREolT0DToZT0mFU2s1atSwOEwv8VTm8uXLLX711Ve9cZm6K/br18973b9/f4ufeeaZcn+2fDd+vzlVAQDPP/98ZZ+O7CQu9ZwxY4bFnP4BgM6dO1vMU+eAn77icVz2CgB9+vSxmFNKYTqCN9Zt1aqVxWHrCD6P1q1bWxx2I+cuzO+88w6kagjTnvXr17eYfxfE1v2Y0038e4k3LQaA3Xffvdy/n631SraUFN9nW7du9Y699NJLFj/22GMWc4o5PF9OUfG9DfhtJi6//HKLmzVr5o1bsWKFxU899ZR3bNmyZTv+R+RJMzkiIiISJT3kiIiISJSKnq7asGGDxf/3f/9n8cEHH+yN4xXZ4QpynubiSo9wmiyTcEqcp+uGDRtW7rmGMk0zAsDs2bP///buPciq6koD+LdHRYm8RCDK+6EoKA+BWBEfKNEgMWIglMUEUIxjGUet0piRSdCgYsqAJCFCwMSxsASioIKADga0AvIqpgihEVAEkZdBHkKap2jMmT8gK9/e9r25fbnP3d+vKlWrPbu7D33vOfdkr73WtnjPnj3esVxsaFmq+O8aVktksyGcFBa/V+fPn29x2K2YhZWPDRs2tJhTVGGVJXdRzrQKhN9T4XXEaQzeFJKrLwH/mo75WixV3GWa05RcqQMAV199dZXHwpRJuePrgjfX5GolwL9G0qWk+Ovw84tTPvx3XL58uTdu1qxZFm/dutXicGNXTmnz+XGncwAYMWKExX379rX4zjvv9MZxtVVFRYV3jJeOnOx1q5kcERERiZIeckRERCRKesgRERGRKOV8TU7YcZTXq3B56NSpU71x3P2Ruz0Cfm4w7KzLUuUxFy9e7I177bXXLF6wYIHF4dod/l1cdj5x4kRvHOc4H3vsMe9YbDllxuWGd999t3cs7HgrpYevkU8//dRi7gIO+Dn7cL3OOeecY/E111xjMe9SDPjXEu8aHjp27JjFfC8J8/J8L+EOuWGZOJ9vbN1zS1FY8s33Xu4+ze0BAKBu3boWh+tTYsKl8rzmLPxcS7UOha9TwN/Ze9y4cd4x/ttzu4/9+/d74zIt0081jjunA8Dw4cMt5jYv4ZqcyspKizdt2uQdy+X6Oc3kiIiISJT0kCMiIiJRyvm8IJegAX4aY+DAgRbPnj3bG7d06VKLn3jiCe8YdzflzTrDKU+eTuOpsJkzZ3rjJk2aVOX3hHgKkX8vd3YFgLZt21r84YcfesdGjRqV8ueXu82bN1v8/PPPe8d+9KMfFfp05CSkSl0BwLJlyywO01VcEstpLk4/AH4LhpYtW6Y8j71791rMGwSm28jzrbfespi7OAP+fUDyL0y78D26du3aFoflybF1Ns4Ev6fDfz9fj3xs48aN3jhOJYfLMriEnH9XrtO2YWqJS9n5nMK2ErwDArcayDXN5IiIiEiU9JAjIiIiUdJDjoiIiEQp77V6XBLKOwSH+Tlu8cwlaIDf5nvw4MEWX3fddSl/L5eJv/DCC96xTPO/nLtcsmSJxeG6I847pytxj9nYsWO9r/v06WOxysnLS5hj53UyHIfWrFmT8hi3d8imhDydmrieo1xwuTKXE4dbe3Bb/3An+Zjwv43Xhp599tneOP7s4XYJvP4M8Lc0CdfSFQuvt+J1mxMmTMjoe3KtZn4ii4iISPT0kCMiIiJRcuk6Czrncrpt7/XXX28xd0IG/HK3TKXrjJnPkjneMR0A7rjjDovDcvXJkydX++cnSZKXLbxz/Xqmw3+jcJryySeftHj8+PGFOqWiycfrWcjXMtcy3YW8FMVwbRYSp+95x/oQp2QKmXYp5rXJbRXSXRNMqdnUUr2WmskRERGRKOkhR0RERKJU0HRVrHhKNvx7ZjP9HtuUeFhJx90ta8KmiUpXxSO2a7Om07UZD6WrREREpEbRQ46IiIhESQ85IiIiEiWtySlByvvHRXn/eOjajIuuzXhoTY6IiIjUKHrIERERkSilTVeJiIiIlCvN5IiIiEiU9JAjIiIiUdJDjoiIiEQpuocc59ww59w7zrkjzrmPnXOTnHMN/vV3SqnRaxkf59wW59y1xT4PEfkn59xU59xO59wB59z7zrn/KPY55UpUDznOuQcAjAbwXwDqA/g6gFYAFjjnahXz3KR69FqKiBTMEwBaJ0lSD0A/AI8757oX+ZxyIpqHHOdcPQCPArg3SZI3kiT5PEmSLQBuBtAawJAinp5Ug17LODnnpgBoCWCuc+6Qc+7BYp+TZMc5180592fn3EHn3EvOuenOuceLfV6SnSRJ1iVJcuwfX574X7sinlLORPOQA6AngDMAzOT/mCTJIQD/C+C6YpyUZEWvZYSSJBkKYBuAG5MkqZMkyZhin5NU34mZ1FkAngPQEMALAPoX85zk5DnnJjrnjgB4D8BOHL/Xlr2YHnIaAdibJMnfqji288RxKQ96LUVK19cBnArgqROzrDMB/F+Rz0lOUpIk/wmgLoArcfz/YB5L/x3lIaaHnL0AGjnnTq3i2Lknjkt50GspUrqaAvgo8TvJbi/WyUjuJEnyRZIkSwA0B3BXsc8nF2J6yFmO40+eA/g/OufqAOgL4K1inJRkRa9lvNRivfztBNDMOccbIrYo1slIXpwKrckpLUmSVOL4YtXxzrnrnXOnOedaA5gBYAeAKUU8PakGvZZR2wWgbbFPQk7KcgBfALjHOXeqc+4mAJcW+ZwkS865Js65Qc65Os65U5xzfQD8OyL5P5PR7V3lnLsdwP04/hR6AMCrAP47SZL9RT0xqTa9lvE58YE4HkA9AI8nSTK2yKckWXDO9QDwPwDOAzAPwCkA/pwkyaiinphUm3OuMYCXAXTB8YmPrTi+3uqZop5YjkT3kCMiIoXlnFsB4OkkSSYX+1xEWDTpKhERKQznXC/n3Dkn0lW3AugM4I1in5dIqKrqFRERkXQuwPE1cmcC2AxgYJIkO4t7SiJfpnSViIiIREkxksC4AAAVOElEQVTpKhEREYmSHnJEREQkSmnX5DjnlMsqgiRJ3L8eVX16PYsjH6+nXsvi0LUZF12b8Uj1WmomR0RERKKkhxwRERGJUtFKyE89NfNf/be/VbUZtUh6mb7H9P4qrH/7t3/+fyt/+yPgiy++KPTpSAp8/Zx11lnesfr161f5PeG1tHXrVotVyVte0t0/y+meqZkcERERiZIeckRERCRKesgRERGRKOV9TQ7n3Fu1amVx//79vXH16tWz+MCBA96xt99+2+LKykqLw7zgrl27LP78889TjpPydvrpp3tfn3vuuRY3bNjQ4l69ennjUr3HZs+e7Y3jdQRaI5Ibp5xyisVXXXWVxRdeeKE3bvr06Rbv27cv/ydWwzVp0sT7+q677rJ40KBBFvN1BXx5jc4/HDt2zPt6yJAhFofXmZQGXiPXunVri2+66aaU3zNr1iyLt2zZko/TyhnN5IiIiEiU9JAjIiIiUcp7uqpx48YWjxw50uKBAwd64zgFwakmIHUaKkxrLVq0yOKPPvqoyv8OANu3b7d4//79FiutVbpq165t8Xe+8x3v2P33329xo0aNLP7qV7/qjTvttNMs5ml1Tp8AwOTJky2eM2dOlmcsjFPV9957r8UdOnTwxq1YscJipavyo1+/fhb/8pe/9I61bdvW4k8//dTi9evXe+OWLl1qcZcuXSxu06aNN+6ZZ56x+O9//7vFc+fOre5pSzVxipFjTh0DQMeOHS0eNmyYxb179/bGcQsA/nnTpk3zxu3du9fiTz75pMrvLyTN5IiIiEiU9JAjIiIiUXLpppCy2WiMV2oD/grtSZMmWRyu6ucqrEyntcJxXAnDaa2dO3d641avXm3x8uXLLV6wYIE37v3337eYp27zTZsAHscpqt/97ncWDxgwIOU4FnbTZeneY5s3b7a4U6dO3rGjR4+m/L40v6vGbQIYdkvlFNVjjz1mcXi/eOihhyweP368xaWSSi7HazO8Pt555x2LOT0FAB988IHFvKRg3bp13ji+trp3727x4sWLvXGcGnn00UerjIsppmuzQYMG3tdjx461ePDgwRaH6Sq+BsPrMRVOPXIM+O8VXkqwdu1ab1yuU1naoFNERERqFD3kiIiISJT0kCMiIiJRynkJeZjT4/K0VDvXAn5OLszPce6OS77T4fxkixYtvGMtW7a0uG/fvhYPHTrUG8ddHTnesGGDNy7s8inZCdcO8NobjlOtwQHSr8NJNS58vzVt2tTi888/3zu2Zs2ajH6+pMb3CC7rB/yu1JIbzZo1877m+2G41pDX4VRUVFgcrqEcPXq0xXwPDddiaefxwglfS/6s5HU4me4unmkrl/AavuCCCyx+9tlnLQ5bufDavHx2TdZMjoiIiERJDzkiIiISpZynq8JysmXLllk8f/58i/v06eONq1WrlsWcngKAESNGWMybdYabJ/KUHKfJOnfu7I3r1q2bxddee63FYblw+/btLeYuu2PGjPHGvfrqqxZnU2Jck3Gn67CT8SOPPGJxuhQV4+nxdGXH6aZszzjjDIu/8Y1veMe482uplDWXovBvw9ctT3s3b97cG8eprExTj5K9w4cPe19z6qFHjx4Wv/DCC964du3aWfzZZ59ZHKbu+b4u+RVec7zR8MaNGy0O7338mf36669bzLsGAKk3yuYu84DfKoY/e8OUJ7cy+NWvfoV80UyOiIiIREkPOSIiIhKlvKereCM37q555ZVXeuM4zRB2RuROxJmuwubpuddee807xpsFciqMqwQAf3NRnnbjVeGAP8X+yiuveMcK2Sm5HHHlB3fHBL7cjTUVfu/w+4PTiOG4q6++2uLWrVt743jzOVX75AZPb3NlRliNedFFF1nMFZJ79uzJ49nFjf/2gL8cINzEdsaMGVX+jDCtyF3BlyxZYvHZZ5/tjbvhhhsszrSbrmQnTFf9/ve/t/iNN97I6GdwiiqsruKfz6nkOnXqeOP4vcLjwtRl+H35onediIiIREkPOSIiIhIlPeSIiIhIlHK+JifE+blvf/vbFp955pneOM4Z/vSnP/WOhaVsmUi3SyrvtDt8+HCLFy5c6I275557LOZ1G23atPHGjRw50uJDhw55x3g9UFjyXlNwSTbgtw+47bbbLA5L+Dmfy+sIuDQSAObMmWPx888/n3Icr8nhc/rxj3/sjQu/lpPH+fyDBw9aHJaJd+3a1WLuzKs1OdkL/3bf/OY3Le7fv793jK/Bbdu2WcxrKwH/fs1l47wGB/BbdPCxUaNGeePUjiH39u3bV2WcLS4959fyySef9Mbxmsbdu3dbPG/ePG/clClTTvqcMqGZHBEREYmSHnJEREQkSjlPV4XdFHk6lKcuuUsm4HdTXLdunXcs11OZnLbg6bQXX3zRG8dTtLfeeqvF9913nzeO01fDhg3zjnFXRy67jB2nIb773e96xzi9x3877lgN+NPgv/nNbyyeOnWqN27Hjh0WZ1qyX1NTh8XC0+WrVq2yuEuXLt64c8891+JevXpZHG6MqvRG9rhFR9iug8u8wzR/JsJSZU55dejQweKwJD2fGzSKL0wRcyn3kSNHLA431L700kst5qUc4fINfs3Hjh1r8UsvveSNK1QKWjM5IiIiEiU95IiIiEiU9JAjIiIiUcrJmhzO8YUt8q+66iqLOd87c+ZMbxxvh1CsfHu4g+6mTZssfvbZZy0O/40DBw60mNcdAcCQIUMs5t3LY9vuIczz8pYMvJt4eIy/L/z7v/zyyxZPmDDB4r17957UuQLAWWedZTFvJRCe04EDB076d4lfVtqtWzeLw3VYvAu2ttQovGzW4bBwrRuvSeS1G+H2D1qTUzh87wOAK664wuKjR49a/L3vfc8bx2uq+BoOP69//vOfW7xo0SKLwzU4vDY2nzSTIyIiIlHSQ46IiIhEKSfpKp56/MlPfuId4/TNzp07Lf71r3/tjQu705YaPr/w3C+77DKLw3K6G2+80WLuxhvb9Czv7A74KapwN3FOB/FOtytXrvTGcflhLlJUnC69/PLLLebpWsCfRuXpVkCly9ni1hJ169Yt4plIPoXproqKCosHDBhgMXe/B4A//elP+T0xMQ0aNPC+5u7F5513nsXhfZGvYX6d3333XW8ct/jgkvRCpadCmskRERGRKOkhR0RERKKUdboqVUUVV1MBfopg8eLFFofpmmJNZWWKzy9Mra1evdrisPKKK0ROP/30/JxckXAa6he/+IV3jKejw8orxtUXvFkn4Fe35QKnVQcPHmxxo0aNvHHcsTOcio0Vvze56zDw5S7mqVRWVlYZA/4UOVdQpcP3Do6B1O+pUr+P1DSzZ8+2mDe+DTcGHT16tMWxVZ6WGt4gF/Crra655pqU38eVcxMnTrT4iSee8MYdPnz4ZE8xpzSTIyIiIlHSQ46IiIhESQ85IiIiEqWs1+Q0btzY4rvvvtvipk2beuM2bNhg8bhx4yz+5JNPsv3VRbd//37v6xUrVlj8rW99yzvGaxE6duxo8caNG71xJ9tptFB4fUaqHeaBL3eyZVyG/cc//tHijz76KBenmBLnni+++GKLw/Ud/N7kEvfY8GvUp08fi0eMGOGNC0tOGb9v161bZ/H69eu9cbzbeLjmh/Ham0suucTinj17euNq1apV5ff/5S9/8b7m6yzsqC35x+8Djjt16uSN413Jc70WT3z79u3zvp4+fbrF/fr1szi8xni9G9/DwzU+pUYzOSIiIhIlPeSIiIhIlDJOV4UlnNwxtm/fvv/8gUG56YIFCyx+//33LS7nUs/w3Hfv3m1xWP7IU/2cIpk7d643rlzSVbzRInd6rl27tjeO/0ZhCoGPLVu2zOJcpxPCNBSnPPjfwV05Af+1ibnDMafveDM+3nwPSJ96ZOeff77FPO0NpC4HD18jPnbddddZ3LVr14zO6eOPP/a+5s7br7/+unesXK65csbXD//9w9eT01dKV+VXuIkqd3V/6qmnLO7du7c3rkWLFhbz9b1q1Spv3LRp03JynrmimRwRERGJkh5yREREJEpZp6s49cLT3mHlUT7TEcUSrjrnv0WdOnW8Y1ydc+jQofyeWAFwapI3cAvfH7yhJlffAX7VzNKlSy3OdfqAKwABf/NYTiOGaQzeYC5mnDbk92aYcs20Uze/BzLtkhymfjl9xb+Xq2/S4a7WgF/VNW/ePO+Y0lWF9de//tXi8H7BHci5S7Jeo/zje/XIkSMtfvrpp71xvJsB30t/+MMfeuNeeeUVi0uhe7VmckRERCRKesgRERGRKOkhR0RERKKUdcfjVGWgnHcF/C6XseRXw46tvHNrWNq6Y8cOi7lUr1xKk8Mdo3kn4SZNmlgcliUuXLjQ4nDNC5do57qVAJ/Tvffe6x1r166dxR988IHFo0aN8sZt3rw5p+dUqrjz6eOPP24xdy4GgHr16qX8GXwfuOiii6qMgdS7nIfrfbJ5P/C1xDvIA0BFRYXFsdx/yhWvteH3G+CXkPN7Sq9ZYR09etTi8D744YcfWrx27VqLf/vb33rjeN3mnDlzLA4/IwpFMzkiIiISJT3kiIiISJSyTlfVNNzRl0vpAKBly5Ypv483eww3RisH4eaM3C6Ahf+2GTNmWJzvaUou27/jjjuqjAH/teAy8a1bt+bx7EoXp4a2bNli8fjx4zP+GZyq5vdK/fr1vXF87OGHH7b4hhtu8MZxqoLbL4Sbt/L7jTd5ffvtt71x3HFdqY/i2r59u8Wcxgf89wffY/bs2ZP/E5OM8P2Cr7+VK1d6426++WaLlyxZYnGxXkvN5IiIiEiU9JAjIiIiUcp5uipMb3Ts2NHijRs3WlwOU8df+cpXLO7fv7/FI0aM8MZxl9V0mz2GmweWg3RpBxZW1b3zzjt5OyeuoAL8tNTtt99ucaNGjbxxU6ZMsZgrAjiNJdlX/vF0dDg1zVV6q1evtpgrMQA/XcUpqjvvvNMbx/eSXbt2WRx2VS+H+0xNka5yLlWqU+mq9LizeHiv5jR+WCXLKisrLc70782/t3v37t4xrsbkTT2VrhIRERHJIT3kiIiISJT0kCMiIiJRyvmanLDEuGfPnhbPnz/f4sOHD+f6V2eFOxS3atXKOzZ06FCLb7nlFotbt27tjeMS6TfffNM7xqXKpbAj68nikuF0cl023rZtW4tvu+027xivyeH1UWHHTu6yunv37pyen6TH6zEyXSfDJeTcbRXwS96lPKS7d/D6q7BdQE3Efyu+p4XtSrjbftiZvE2bNhZ/7WtfS/m7uCs4d3/nkn/AXzfF63CaN2/ujeM1jldccUWVvwcoXAdkzeSIiIhIlPSQIyIiIlHKOl3FU848FR2Wqg0YMMBi3vgv7Ey6c+dOi3maujq49JVLUcNNAHmDwG7dulk8ePBgb9y1115rMZeTh2Wqf/jDHyyeMGGCd6ymdtNNJ1WK8KabbvLGNWvWzOIhQ4ZYHE7LbtiwweLhw4dbPG/ePG+cUlSlIdW9Q+LWoUMHi7m0GPDvk9ne/2PC98Xp06dbfMkll3jjuJQ7XRqYPw/DpSL8efbII49Y3KNHD29cw4YNLebr9swzz/TGPfPMMxZrg04RERGRPNFDjoiIiEQp43RVOBXGXUt5qrFdu3beOK5EGjNmjMXbtm3zxi1cuNDigwcPZnROBw4c8L5evHixxTwdyhVegL8incfxdFyIuzWGaZCf/exnFodVIMWaoiu0sHKC03vt27f3jnXu3NliThFyehDw04y8IRxP3wLAuHHjLH7vvfcsDtOKUjx8/1i7dq3FYafsxo0bW8xT8dy9VcoH3xd46cIZZ5zhjeMO6epS7eNUE18TgP/35WUAoXTppYsvvthirsiqW7duRucXfsbxs0FYoVUMmskRERGRKOkhR0RERKKkhxwRERGJUtZrchYsWGDxyJEjLX7wwQe9cRdeeKHFvCt0uEM0r9PIVFhqyLsRc94x7MLMuUv+d4W7UfManxdffLHK/w7463ViL4lN9e8L1zPdf//9Fl922WXeMX49+Ps49wz4f9d0peE16e9frvg64zx9WM7Ka3J4HQ6vFQCANWvWVPmzpbjCdSGDBg2ymD8bwtedu5Hr9fQ/yxYtWmRxly5dvHHpdhdn6bpN830303U46T43+etSuB9rJkdERESipIccERERiVLWHY95s8mZM2davH79em/cfffdZ3Hfvn0tTpdCCtMWqTqkhmWI4UZh/xCmtbh8fdWqVRaHpcmcluIpuJpSFg4AlZWV3tdc8suphTBdxR2K05U2cpk3l38Dfmk4pwtVGl5++D3QtWtXi3nTP8C/vjldxRu0Av49QumN7IXpDu4yvmPHDou5mzwA1K5d22LuIP/www974zhdxa/tD37wA28cl5CL/5nF9+Dws4dLysPUUKoUVbabLPNn4JIlSywOdy/gY6VwbWomR0RERKKkhxwRERGJkh5yREREJEouXYmXc67a9V9hvo/XbVx++eUWhyXjnTp1sph3qwWAd99912JeT8O5fcBfX8NbQ4TbP3BJHpez8tYBQPHW3iRJklnStJqyeT3D9VFPP/20xf369bM4XGPF74NwDc2bb75p8XPPPWcxtyUAgCNHjlT3dEtSPl7PbF7LUsFbvTz00EPeMV4Twuu8pkyZ4o2bNGmSxYW8Tkvp2syF8847z/t65cqVFm/atMnicN0FX/u8JqdWrVreON7y54EHHrCYd6cGildqXA7XZsuWLS3u3r27d4yvpXDd1Pe//32L+XUO76u8zrKiosLiclujmuq11EyOiIiIREkPOSIiIhKlnKer0uHUR1hWzKWk9evX945xCR13ygxTJJxuCsvGWTitV2pKeUq8SZMmFnP6kXeyBfzXOkwXzp4922Kezi6Vac9cK4cp8ULiVGbYeoDTHbwLffgeClPLhVLK12Y2+G8M+J3Fb7nlFovDTrifffaZxStWrLB42rRp3jhOcezdu/fkTjYPyu3aDJcPpNuhvH379hb37t3bYr7nAn7bF76uSmX5RqaUrhIREZEaRQ85IiIiEqWCpqskM+UyJc5TpeE0ajqlni7MtXKbEpfUyuXazAXuaszdpwF/OQBXspZ6SiNUU65NXh4SfuaXQlfiXFC6SkRERGoUPeSIiIhIlPSQIyIiIlHSmpwSVJPy/jVBTcn71wS6NuOiazMeWpMjIiIiNYoeckRERCRKadNVIiIiIuVKMzkiIiISJT3kiIiISJT0kCMiIiJR0kOOiIiIREkPOSIiIhIlPeSIiIhIlP4fsjncFxzaZ4QAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x720 with 25 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TiVoXehQnsOB",
        "colab_type": "text"
      },
      "source": [
        "#### Get all the images in an array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3BZDQWDjRlXv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "6d141879-4094-47eb-f0e9-2ca307e74def"
      },
      "source": [
        "IMAGES = []\n",
        "LABELS = []\n",
        "\n",
        "for img, label in tqdm(trainloader):\n",
        "    IMAGES.append(tf.reshape(img, (28,28)).numpy())\n",
        "    LABELS.append(label.numpy())\n",
        "\n",
        "IMAGES = np.array(IMAGES)\n",
        "LABELS = np.array(LABELS)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "593242it [08:06, 1219.23it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "75WCj0tmqIwo",
        "colab_type": "text"
      },
      "source": [
        "#### Index Images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Th2nVHDvVYiG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d3b40d7a-4770-48dc-bf9b-42f2555d5508"
      },
      "source": [
        "image_index = {}  # where key is the char and value is a list of IDs\n",
        "for idx, label in tqdm(enumerate(LABELS)):\n",
        "    char = MAPPINGS[label]\n",
        "    if char in image_index:\n",
        "        # this character already exists\n",
        "        image_index[char].append(idx) # append index\n",
        "    else:\n",
        "        image_index[char] = [idx]  # initiate list with 1 item"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "593242it [00:00, 1426612.32it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jMWtqzPdqbO_",
        "colab_type": "text"
      },
      "source": [
        "# Utils to generate image from text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A0bAc0kPXf9j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_sample_sentences(sentences_in=sentences, num_sentence=10):\n",
        "    # Get a defined number of sentences from the data\n",
        "    return np.random.choice(sentences_in, num_sentence)\n",
        "\n",
        "def get_generated_image(words, chars=IMAGES, index=image_index):\n",
        "    # words is string of char/numbers that needs to be converted into an image\n",
        "    # chars is a data set of images that need to be used to compose, usually pass in train['features'] in here\n",
        "    # index maps a character to indexes in the images, available as dictionary\n",
        "    height, width = IMAGES[0].shape # height and width of each character\n",
        "    length = len(words) # total number of characters in the image\n",
        "    \n",
        "    # create an empty array to store the data\n",
        "    image = np.zeros((height, width * length))\n",
        "    pos = 0  # starting index of the character\n",
        "    \n",
        "    for char in words:\n",
        "        if char is ' ':\n",
        "            pos += width # if space, move over\n",
        "        else:\n",
        "            if char in image_index:\n",
        "                # pick a random item from all images for that char\n",
        "                idx = np.random.choice(image_index[char])  \n",
        "                image[:, pos:(pos+width)] += chars[idx]\n",
        "            elif char.upper() in image_index:  # to remove characters from other languages\n",
        "                # for some characters, there is only upper case\n",
        "                idx = np.random.choice(image_index[char.upper()])  \n",
        "                image[:, pos:(pos+width)] += chars[idx]\n",
        "            \n",
        "            pos += width\n",
        "    \n",
        "    return image"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ASm9g8-_qsFR",
        "colab_type": "text"
      },
      "source": [
        "#### Check one sample"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9buIC1YVX1g4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 76
        },
        "outputId": "ced0f134-9fb5-4f34-d02b-3b96569ca113"
      },
      "source": [
        "# Let's print a sample to see how it looks\n",
        "s = get_sample_sentences(sentences, 1)\n",
        "print(s)\n",
        "img = get_generated_image(s[0])\n",
        "plt.figure(figsize=(9,2))\n",
        "plt.imshow(img, cmap='gray')\n",
        "plt.show()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['The poor soils of New Caledonia promote a restriction in large ground dwelling prey']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhgAAAApCAYAAAB3PVAxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd40lEQVR4nO2deXgURfr4PzVXJjOZJJNjwoScwAYIh3IJhICKysoX2MCKK+gKouuBwipCBEUFdHFxld1lfVQQFkFBEBTWsIgSDhVQYEG5CZckIZADwpE7YTL1+2OS3ow50QjLz/o8Tz/pqe6uerveqrfefqvSLaSUKBQKhUKhUDQnumstgEKhUCgUiv//UA6GQqFQKBSKZkc5GAqFQqFQKJod5WAoFAqFQqFodpSDoVAoFAqFotlRDoZCoVAoFIpm5yc5GEKIO4UQR4QQx4UQU5pLKIVCoVAoFNc34se+B0MIoQeOAncAWcB/gJFSykPNJ55CoVAoFIrrkZ8SwbgJOC6l/F5KWQEsB5KaRyyFQqFQKBTXMz/FwWgJnKrxO6sqTaFQKBQKxS8cw89dgBDiEeCRqv1urVq1wmq1UlBQgNvtJi8vj7KyMnQ6HQEBAVy4cMHr+uDgYOx2O7m5uRQWFjapTKfTidlsJjMzk8rKSi1dr9cTHh5OUFAQ33//vVd+Pj4+OJ1OioqKOHfuHDqdDl9fX2w2GyaTidzcXMrLyxssV6/XEx0djclk4sKFC+Tm5gJgMBgICQnBarVy9uxZCgoKtGt0Oh1Op5OcnBwvWQ0GA8HBweTl5SGlRAhBVFQUhYWFnD9/vkn1IIQgMjISi8VCUVEROTk5uFwuAIKCgpBSYjAYKCkpobi4uNH8dDodbrcbHx8fysvL0el02Gw2fHx8uHjxIhUVFRiNRiwWC5cuXQLAz88Pk8nUoMw6nY6goCCsVqvX/QkhMJlMSCmpqKiodV1gYCAAFy9e1NKCgoLw9fVFp9ORl5dXS2dGoxG73U5eXp5X+SaTCZ1OR2lpKaGhoVitVjIyMggPD8fPz49jx45p+jGbzQQEBGj6/TGEhYVRUlKCXq/HaDRy9uxZwNOGYmNjMRg8XTMjI4PS0lKtPiwWi9YfLl++jE6nQwhBSEgIFouF7OxsysrK0Ov1tGzZksDAQMrLyzl+/DiVlZXa+RaLBaPRSEFBgdYmDAYDUkqvdlgfBoMBh8MBoF1fs05/iK+vL6GhoZSVlWEymSgsLNTaiMViISoqCrfbzYkTJ7TyrVYrdrsdnU5HYWGhZhsMBgMtWrQgLy9PaxdWqxUfHx8uXbqEXq9HCEFFRQXVU8CBgYHY7XZOnjypyaTT6YiKisJut5OdnU1OTk4tmV0uF5cvX/a6xuFwkJubq+Xt6+uL0+mkpKSE8+fPazIZjUaCgoJwu90UFRVpemwIIQTR0dHk5ORQVlZWq86DgoIoKSmhqKio0bwiIiIoKCjwsjd6vZ6QkBBMJhNZWVlIKTGZTLRq1YrKykpKS0txu92cOXMG8Nhfs9mM0WgkMzMTt9ut5VNtmzMyMmrJWtd9Vbf5wsJCak7NV9vHyspK8vPzvcpwOp1IKTl9+jTgqWu9Xk9RURF2ux2Xy0VFRQV6vR6Xy4XFYuHy5ct12jNfX18qKyvrtCU/PM/hcJCZmanVj16v1/RnNBoJDg4mJydHaw++vr6UlpbWakOBgYGUlJTUKtNms2E2m7V+X21LjUYjLpeL8vJyrbxqe1NUVKSNLQAOhwOj0QhAUVGR1p8awmg04nA4KC8v59y5cwCajdXr9RgMBoqLi7WxweFwYLVaKS8vJysrC7fbjdFoJDQ0lLNnz+J0OsnOzqaiokLUVd5PiWCcBiJr/I6oSvNCSvmOlLK7lLK7EIJ27drx3HPPkZ2dzalTp7SG2aJFC/r06YNer9eu9fX1ZcGCBdx7773Mnj0bq9XaqFB6vZ6XXnqJfv360atXLy3d39+fZcuW8e2337Jz504SEhK0Y926dWPWrFn07duX++67D4AhQ4Zw4MABBg4cyIABA5gyZQpC1FmHGsHBwcyYMQM/Pz+cTifg6TwzZsxgzJgxOJ1OZs2ahY+Pj3ZNUlISM2bMqJV3nz59mDt3LmazGfA0gmeeeYbf/e53+Pr6NloPAIMGDWLZsmU4nU6GDh3KDTfcAHjqevv27ezZs4ennnoKk8nUYD5+fn4888wzzJ49m27dunHLLbfg7+/PnDlzmDRpEk8//TSTJ0/W6rJHjx5e1w8bNqzeutPr9Tz55JP07t2bBx54gLi4OK+6SU1NZc2aNSQlJXnlYTAYmD17NiNHjtTSYmJiePTRRykpKWHgwIG0a9dOO+bv70/Xrl35/e9/T6tWrbT7GjNmDFOnTmXkyJEMGjQIo9HIxIkTWbVqFVOnTuXw4cNMnDjRS+bevXszatQo7XdAQADJycnExMQ0WI81uXDhAnfddRcmk4mxY8dq6f3792fLli306dOHqKgooqOjtTKef/55Nm/ezLx58/D396d79+6sXbuWNWvWkJCQQOvWrZk8eTIGg4HXX3+dOXPmEBcXx8KFCxk+fDhhYWHMnDmTv//973zyySeMGjUKs9lMaGgo8+bNY/fu3Xz11VfEx8c3KLteryc5OZlu3bpRVFREUlIS48ePR6er35z85je/Ye/evTzyyCMYDAYMBgNCCAYNGsSuXbtYvnw506dPx2KxaNc89NBD3H///fTs2VNr80IIxo4dy9atW/n000+1Pm6xWBg9ejSLFi1i+fLlzJo1S3NAwdM2unbtqrUhIQTjx4/n4MGDmkw1EUJwyy230LKld1A2NDSUMWPGaL9NJhNvv/02y5cvZ/78+axfv57ISI9Z7NKlC3PnzmXw4MH85S9/wW63N1iv4HE833rrLa96GDZsGGPGjGHx4sWsXLmSP//5zw3WNXgcrldffdXL1phMJl588UVGjBjB0qVL6dChAwDt27dn/fr12O12ioqKuPnmm9HpdBiNRl544QWCg4P58MMPCQ8PBzyD5oIFC5g8eTKPPfZYo7KARz/jx4/nzTff1GSyWq04HA7NrhQWFmrOhdVq5dlnn2XYsGHMnTuXkJAQzVb4+fnh7+9PcnIybrebyMhI5s+fzyuvvELHjh2ZMGGCpoOa9O/fv1HbabVaGTJkCMuWLSMqKgqAqKgokpOTSUhI4LbbbuP222+nX79+CCEYPXo0s2bNIjg4mDfeeMNrjLLZbDzzzDNeY1o1o0aN4qmnntLao9vtpqKigl69emE2m72c0aSkJD744ANmzZrFiy++iBACp9PJSy+9RHFxMb/61a949dVXG9WDTqdj3LhxdOzYkSFDhmhpM2bMYOXKlcyfP5+lS5fSvn17rFYrixcvZsyYMVitVl577TVCQ0Mxm80sXLiQ6dOns3LlSvr16+flgNcqs0GJACFEpBBisxDikBDioBDiyapDA4HbqtL3Ag8DKQ3l5Xa7iYqKIjU1lfLyci8vNjAwkJiYGCZNmqQNAOBxMu6++26ysrIAvDpMXRgMBsLCwsjLy/PKp3PnzkRFRTF27FhOnTpFRkYGAHFxcSQlJTFz5kwCAwM5cuQIOp2O22+/ndTUVN5//33effddZs+eTUMLYiMiIrDZbLhcLkwmE3v27AEgISEBHx8f3nnnHdLT00lPT9ee0Gw2GyNGjGDFihXaUyB4lN6lSxe++eYbzQErLy/nT3/6E3379mXKlCmNdhQ/Pz/uv/9+Hn30Ub788kstAmM0Ghk9ejSnT58mJSWFrKysBiMzNpuN559/nrCwMPLz8xk4cCDbt2/npptuIiYmhpiYGMrLy/nggw+0cnU6HTqdDj8/P0pLS9Hr9fU2/uDgYKKjo9mwYQPZ2dkcPXoU8AxicXFxJCcnM3XqVO655x4vY2+z2YiIiODw4cPa+c899xxDhw5l0aJFtG3bVntadTgc3HfffXTq1IkBAwawf/9+hBAMGTKErKwsFixYAMCWLVuoqKggLS2NNm3aMGjQIE6dOsXcuXO9nupbtmzJunXrtN/33nsvsbGx9O3bF19fX9q0aaM93TdEYmIiTz31FN988w3gGdTuuOMOVq1axXvvvcf69etJS0vDYDAwYcIEysvLueuuu3j55Zc5f/48ISEhfPHFF+zevZuYmBiOHj2K2WzGZrMRExPDhAkT+Oijj3jttdfo0qULeXl5TJs2jeTkZLZt28ann35KUVERQgji4uJISUlhyZIlVFZW0r1793qdwoCAAMLCwvj8889p3749ISEhOJ3OOg1pNa1ateLQoUP84x//ID09nfz8fKSU3HrrrcTGxrJ//37GjRvn9WQ+b948Fi9eTHx8PDabTUuPjY3lX//6F6NHj2bv3r0AnD17ln/+85+0aNGC559/nkmTJmlPekII0tPTWb16tdaHg4ODeeyxx0hNTSU5OdkrGhUVFcWkSZMYPXp0rXtq06YNJSUlWj4RERFER0czYsQIZsyYQVBQEGFhYQD06tWL7Oxsli9fTkVFBaGhoY22iQ4dOpCTk6M9jRqNRkaMGMG0adNYt24dR48eZdeuXdpAXB8xMTHk5OSQn5+vpYWHhxMdHU1ERAQbN24kMzMT8PSlnTt3smfPHi5evMiyZctwu924XC4OHDjAE088QU5Ojva0PWrUKG3AnzNnDiUlJY3e1x133MG6devYuHGj9kDTsWNHZs2aRWZmJuvWrdNsnRCC3/72t1y4cIEvv/yS/Px8iouLiYyMJDc3l/z8fEaOHMn27dspLi7m1KlTWK1WFi1axNatWzlw4ID2MAVgt9txOp20aNGi0Sj4iBEjaNu2LV9//TX9+/cHICsri8jISB588EEefvhhbrrpJvbt26fpJjo6mscff5xt27Z53cPw4cM5duxYrQFYr9fTpk0b1qxZ4zWmlJaWIqX0qk+TyURSUhLffvstqamp7Nu3DyklPXr0ICMjA51Ox6xZsygoKGhwfAKP/hMTE4mPj2fjxo2Apz0PHTqUDRs28Nhjj/Hkk09y4cIFBg4cSI8ePTAYDLzwwgt899135Ofn43K5WLFiBf7+/qxdu1az+/XRlAiGC5gopYwHegFPCCHiATewCM80ix/wTynlwYYyEkLQt29fdu3aVeuY3W4nPDycDz/8kE6dOgGeQfXo0aMcPnyYM2fOMGXKFO1YfVRWVlJZWcmUKVO8wlJdunRh9+7d9O/fn8WLF3P8+HF0Op3mzUdEROBwOPjyyy8xmUzceOONfPXVV7hcLiorKxsNSQ4bNozu3buzc+dOHn74YXx8fLQBY/Xq1XTu3Jmnn36azz77THMmEhISyM3NZfPmzV55OZ1ObrvtNlJSUrwazblz53jllVfo0qULd955Z4PytGvXjuLiYo4dO4bD4SAuLo5Tp07Ro0cPHn74YbKysti7dy8ff/xxgwbi17/+NVu2bOHll18mJyeH5cuXc+nSJc6ePct3333HkiVLSE1NpWXLlsTExFBZWYmUko4dO/Lggw9iMpkoKyur1yBWh/Oro1UXLlzAYDDgdrvJzc1l6tSpTJs2jdjYWC/d9+zZEz8/P44cOQJ49L57924yMjJYuXIlW7duZcyYMRiNRm655RZ27tzJ6dOn2bt3L8XFxdqUU2xsLCNGjGD//v1aeDMwMJD09HS2bdvGkSNHOHDggFauwWAgNjZWc1ABiouLOXjwoDYw3XbbbY12dikleXl5vPHGG6SmpgIeQ9+jRw8++OADL6PUpk0b2rVrx7Zt22jTpg07duzQQpp5eXls2rQJPz8/LBYLZWVlOJ1OdDodYWFhtGvXjqFDh/L9999rU016vZ6KigptgDl79ixz584lKiqK+Ph4Jk6cSKdOnep1MKrLNhqNZGVlER4eTkZGRoNPMna7nR07dtQy8KdPn2bevHnMnDmT+Ph4LaJms9no3bu3FuKvdoKllJw/f549e/ZgtVq9QvMGg4HDhw9z8uRJL4fQYrHw6quvekU0QkJCcDgcnD59mkcffZSZM2dqTnu1TGvXrvXSsxCCxMREduzY4ZV3YGAg48eP5/XXX2fZsmWaA9u6dWs++eQTXC4Xfn5+TZrWuPHGG9myZYsmf0BAAHFxccyZM4etW7fWcm7ro3379hw/ftyr35WWluJ0OunWrRtvvvmmNnUSHR3NwYMHa4XxpZQsXLiQF198kZMnT+JyuTAajdx8883MmDGDSZMmkZmZSXh4uFfEpS6qB0/wPISAx753796dNWvWeMnp5+dHUlISRUVFvPDCC3z88ceUlZXRu3dvjh07xvDhwzGbzaxduxbw6KWkpESboisuLtacvG7dunHffffxxz/+kdzc3EYdM4fDwbp167h06RLdu3dHr9dTVlZGZmYmTqeTDRs2MHz4cG2KPysri7S0NFauXMnHH3+sRal69uzJAw88QHx8PMnJyfTq1UvrT506dcJsNrNv3z6vsvV6PZGRkfTr14/u3btjNpuJj4/Hx8eHv/71ryQmJnLqlGfZo6+vrxaB9ff358MPP2zU5sTExHD8+HGWLVum9f1BgwaRkpLCggULKC4uJj09nezsbPR6PV988QWZmZm8++677NixA4fDgcvlorS0lP3797N48eJG67PRNRhSymwgu2q/UAhxmP8u5kyTUv6hsTyqMZlMtGjRwqvTVpOfn897771HWVmZV9ho3rx5vP3224wbN47NmzeTlpbWmLykpaWxaNEir3ICAgK00Nfrr7+Oy+XCYDBgsVjo378/ERERzJ8/n9LSUoxGIykpKZqX1xQ2b97M5MmTiYiIwGw2ExwczJkzZ0hPT2fq1KlkZGTwxhtvcOjQf/+Lt23btixdurSWYbZarXz77becOHHCK71t27Y8/vjjSCm95pLrIisri6KiIl577TUuX77Mu+++y6VLl7Db7YwfP55NmzY1uqYEYPv27dpcd/WTPsC+ffu4fPkyiYmJBAUFce7cOdLS0sjOzmbw4MFUVFQQFhbGE088wdatW+tt/IWFhfz73//mySef5PTp02RkZJCRkUFaWhofffQRO3bsoKKiguLiYi8D7XQ6Wblypde8/8aNGxk8eDBPPPEERqORd955R5tX7tOnD71792bDhg2Ap20tWbKE1q1bc+jQIW0+EmDVqlWkpKRw5513YjQavfRT/SRdc473/fffBzyGTqfTeUWj6kNKqTlq1fj5+XHixAkt+lVNQUEBOTk5dO7cmVWrVml1uWnTJm3dwZYtW2jbti179uzh/Pnz2O12Vq9eTWFhIZs3b+ajjz7S8uvQoYNXdExKyYoVK9i8eTNBQUHk5eVx4cKFenVWUFDAsWPHmDZtGkIIUlJStPB5Xeh0Onx8fNi0aVOtPA8dOsQ999xDt27dyM7OZsmSJVy+fBm73c7YsWM1415zbvurr75i9uzZtGvXjqlTp2qDcXl5OWvWrKm1HqCiokKbv64mPT2d9evX88ADD5CVlcX48eO1sHRlZSUlJSWcPXu2VmQxPDzca51YZmYmW7duxWAwMHHiRPbu3avZlhMnTrB9+3ZMJhMWi6XRNRjV6y+2bt2qpRUVFTF58mS2bNkCwLPPPuvVZurD19dXi/pWU/1QcOzYMa/2brVaSU9Pr1OeW2+9Vbtnt9uN2+3m888/Z9q0aZw4cYKLFy9y7tw5Fi9e3KA8X3zxBX/4wx8IDAzUHJlTp07x2WefaesrapZbUFDA8OHDWblyJZ999hlSSlq0aEFCQgJpaWnMnz9f07vb7Wb79u2afTh58iQGgwGTyUSfPn04dOgQrVu3rvPB9od8/fXXjBkzhm+++Uazl+fOnWPVqlWUlZWxevVqevbsSVFRES6Xi5kzZzJhwgTGjRvHkSNHeOuttygpKeHOO+/koYceIj09HZvNpk3ZVNfpihUrtAc7Hx8f+vXrR0BAgOY8nzlzRov4/+1vf+P8+fPo9XrNNmzZsoXExEQOHTrEsmXLGp2+F0LQp08fli5d6tWXPv/8c23dVk0+/fRTbr75ZgYMGIDFYuHAgQPs2rWLuLg4Bg8ezPTp0xtdywJ4jEtTNyAGyAT8gelAOrAPWAjY67nmEWAXsEsIIePj46VOp5NAnZvNZpPx8fFeaYGBgTIkJEQaDIZ6r6vehBCyZ8+e0mg0eqVHRkbKXr16SR8fH6/0mJgY2bNnT2m1Wr3SG5Kxvs3X11e2bNlSOp1O7Xqj0SjDw8OlyWSqdb6/v3+d5ej1emmxWOq8N6vVKoOCgpokj7+/v7zhhhtkWFiYlmYwGKQQ4orvrambwWCQXbt2lYGBgTIoKEjGxcU1qS5tNpsMDAyspbeG6vqHugSk1WqVUVFRMiwsTCvXZDLJxMRE2adPn1p6bmgzm80yODi4VlpcXNxPrichhOzVq5eXLgwGgwwJCanz/KbUoRBCO8/hcMiuXbvKdu3aSbPZ7HVeaGhorbQr3YxGowwLC5N+fn7SYDDUqqcfbsHBwXXqy2QyyYSEBNm5c+dafcTf31+2bdtW2mw2r3S9Xi87dOggAwICmixvTExMLd3b7XbZr18/GR0dXet8vV4vnU5nrfQWLVrUaqN6vb5O/VTbK51OJ1u3bt0kHcbHx0s/P7+f3L4CAgKkw+GolR4YGCh9fX290qKiomSrVq3qbE+JiYly5MiRMioqyut+W7duLTt27CgdDkeTbWVgYKAcOXKklx2vq01Uty+z2ezVP1q1aiVvvfXWOm3pD2UQQki9Xi/vvvtuOWjQINmxY8cmySiEkBaLRer1eunv7+/VT6rl/mE7MhgM0m63a+cKIWrVcc0tKCiolrz12WW9Xi+FEFII4TWu1DwWFhbmpZ+6Np1OJxMSEuqsu/o2Hx8f2aFDBxkTE6Pde7du3eosqz6fockv2hJC+AFfAjOllKuEEGHAuaoCXgacUsoHG8mjEDjSpAIV/wuE4NGx4vpA6ev6Qunr+kLpq26ipZR1LjBqkoMhhDAC/wY+l1L+tY7jMcC/pZQdG8lnl5Sye1MkVlx7lL6uL5S+ri+Uvq4vlL6unKb8F4kA/gkcrulcCCGcNU4bBhz44bUKhUKhUCh+mTTlRVt9gPuB/UKI6tVnzwEjhRA34pkiSQce/VkkVCgUCoVCcd3RlP8i2QrUtUT10x9R3js/4hrFtUPp6/pC6ev6Qunr+kLp6wr50V9TVSgUCoVCoaiPn/KqcIVCoVAoFIo6uWoOhhDiTiHEESHEcSHElKtVrqJ+hBDpQoj9Qog9QohdVWlBQohUIcSxqr/2qnQhhPhHlf72CSG6XlvpfxkIIRYKIfKEEAdqpF2xjoQQo6vOPyaEGH0t7uWXQD36mi6EOF3Vz/YIIf6vxrFnq/R1RAjx6xrpyl7+zNT3GQzVv5qRK3nR1o/dAD1wAmgFmIC9QPzVKFttDeolHQj5QdpfgClV+1OAV6v2/w9Yh2c9Ti9gx7WW/5ewAf2ArsCBH6sjIAj4vuqvvWq/zhfjqe1n0dd0YFId58ZX2UIfILbKRuqVvbxqunICXav2bcDRKp2o/tVM29WKYNwEHJdSfi+lrACWA0lXqWzFlZEEVL/3dzEwtEb6e9LDdiDwB/+qrPgZkFJ+BfzwW/dXqqNfA6lSyvNSygtAKtDwx2wUP4p69FUfScByKWW5lPIkcByPrVT28iogpcyWUn5btV8IVH8GQ/WvZuJqORgtgVM1fmfx3++ZKK4dElgvhNgthHikKi1Mer4/A5ADhFXtKx3+73ClOlK6u/aMqwqrL6wOuaP09T9D1csiuwA7UP2r2VCLPH/ZJEopuwID8Xwlt1/Ng9IT/1P/ZvQ/jNLRdcHbQGvgRjwfjpx9bcVR1KTqMxgfA09JKQtqHlP966dxtRyM00Bkjd8RVWmKa4iU8nTV3zxgNZ7QbG711EfV3+rPlSod/u9wpTpSuruGSClzpZSVUko3MB9PPwOlr2tO1WcwPgaWSilXVSWr/tVMXC0H4z/Ar4QQsUIIEzACSLlKZSvqQAhhFULYqveBAXhe954CVK+CHg18UrWfAoyqWkndC7hUI4youLpcqY4+BwYIIexV4fkBVWmKq0ADn1VIAUYIIXyEELHAr4CdKHt5VajvMxio/tVsNOVV4T8ZKaVLCDEOT6XrgYVSyoNXo2xFvYQBqz19DAPwgZTyMyHEf4AVQoiHgAzgd1Xnf4pnFfVxoAQYc/VF/uUhhFgG3AKECCGygGnALK5AR1LK80KIl/EMXAAvSSmbuhBRcQXUo69bRB2fVZBSHhRCrAAOAS7gCSllZVU+yl7+/NT3GQzVv5oJ9SZPhUKhUCgUzY5a5KlQKBQKhaLZUQ6GQqFQKBSKZkc5GAqFQqFQKJod5WAoFAqFQqFodpSDoVAoFAqFotlRDoZCoVAoFIpmRzkYCoVCoVAomh3lYCgUCoVCoWh2/h8G1qv9FTohugAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 648x144 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7EzTHffnrEo4",
        "colab_type": "text"
      },
      "source": [
        "# Build Training Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1IEJhTF3YEg4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_sentences = sentences[:2000]\n",
        "test_sentences = sentences[2000:2500]\n",
        "\n",
        "# Lets assume that for each training sample, 2 variants will be generated\n",
        "\n",
        "def generate_sentences(texts, chars, \n",
        "                           index, num_variants=2, max_length=32):\n",
        "    # this method takes input text lines, character samples and labels\n",
        "    # and generates images. It can generate multiple images per sentence\n",
        "    # as controlled by num_variants parameter. max_length parameter\n",
        "    # ensures that all sentences are the same length\n",
        "    \n",
        "    # total number of samples to generate\n",
        "    num_samples = len(texts) * num_variants\n",
        "    height, width = chars[0].shape  # shape of image\n",
        "    \n",
        "    # setup empty array of the images\n",
        "    images = np.zeros((num_samples, height, width * max_length), np.float64)\n",
        "    labels = []\n",
        "    \n",
        "    for i, item in tqdm(enumerate(texts)):\n",
        "        padded_item = item[0:max_length] if (len(item) > max_length) else item.ljust(max_length, ' ')\n",
        "        \n",
        "        for v in range(num_variants):\n",
        "            img = get_generated_image(padded_item, chars, index)\n",
        "            images[i*num_variants+v, :, :] += img\n",
        "            labels.append(padded_item)\n",
        "    \n",
        "    return images, labels"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "317adGptYcs5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "caf3ab5e-1e4d-45c9-f437-435aaa24eff5"
      },
      "source": [
        "train_images, train_labels = generate_sentences(train_sentences, IMAGES, image_index)\n",
        "test_images, test_labels = generate_sentences(test_sentences, IMAGES, image_index)\n",
        "\n",
        "train_images, train_labels = shuffle(train_images, train_labels)\n",
        "test_images, test_labels = shuffle(test_images, test_labels)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2000it [02:28, 13.43it/s]\n",
            "500it [00:37, 13.44it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "jgEJHI4CsUvY"
      },
      "source": [
        "#### Check one sample"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZvWNvk4Y1z_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "6bd80504-953a-47eb-e7bf-6f46cd11eb9b"
      },
      "source": [
        "# Sample some to check\n",
        "idx=0\n",
        "print(train_labels[idx])\n",
        "plt.figure(figsize=(9,2))\n",
        "plt.imshow(train_images[idx], cmap='gray');\n",
        "plt.show()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Their music is also featured in \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhgAAAAyCAYAAAAeLKJpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deVhV17m433Vm4ABHUOYpiuKAQ5xRUKMCxliNQ4ya29iYa02uqbFJWrXe+jQxv8YMtqZJmxiHGKNRr2kcYpznOKA4oYCgIAgKMg+CzGf//jhDJQIeEEil+32e88Bee1rf3mv41vd9a20hSRIyMjIyMjIyMs2J4ufOgIyMjIyMjEzbQ1YwZGRkZGRkZJodWcGQkZGRkZGRaXZkBUNGRkZGRkam2ZEVDBkZGRkZGZlmR1YwZGRkZGRkZJqdR1IwhBBjhBCJQogkIcTC5sqUjIyMjIyMzOONaOo6GEIIJXANCAduAdHAdEmS4psvezIyMjIyMjKPI49iwRgIJEmSdEOSpEpgMzChebIlIyMjIyMj8zijeoRzvYH0+7ZvAYN+epAQ4tfAr82b/R7hfvUihEBekVRGRkam5ZHbW5mfIkmSqCv9URQMW2/8BfAFgBCi2Uull5cXo0ePZsOGDRiNxua+fL2oVCqee+45CgsL2bt3b6MrnEKhwM7ODqPRSGVlJTU1NS2U08cDIQQ6nQ6tVktpaSlVVVU2nafRaBBCUFlZ2eh3oFarefbZZ+nVqxe7du3i3LlzrfoedDodNTU1Nsv6UxQKBVqtFiEECoUCBwcHiouLKSsra+actgwGg4FBgwYRFxfHrVu3GnWuTqfDxcUFPz8/2rVrx8WLF8nKyrKpDAghUKvV1NTUtHq9E0IwZMgQwsPDiY+P5/z586SlpT20DNjb2+Pl5UXnzp2xt7cH4OrVq8THt65H2sXFhQ8++IDf//735Ofnt8o9VSoVOp0OSZKoqKigurq6ydfSarX4+PgQGBhIUVERCQkJFBYW2ny+Tqdj2rRpXLlyhQsXLrS6oqXRaPD09CQ7O7vBeq5UKvHy8qJXr16kpqYSHx+PUqnEYDDg4uKC0Wjkzp07lJSUtGh+H0XBuA343rftY05rNpRKJUqlssGGQKVSYTAYHvk+rq6uuLi4UF5eTkZGBpWVlQ2eI0kSTk5OODo6Nlqj1+v1hIWFMWPGDMrLy0lMTCQxMZH4+HgyMjIoLy//jxsh+Pv788wzzxAQEMCJEyc4cuQIxcXFDZ7j4OBAREQEKpWKEydOcOfOnUY9N1dXVxYuXEivXr145plneO211zh9+nSrPHu9Xs8rr7xCVVUV27dv5+bNmzafq9PpCAwMJCwsjCFDhqDValEoFLRv356EhAQ++ugjkpKSWjD3j45CoWDUqFGsWLGCd999l5UrV9p0nr29Pb6+vgwbNoxhw4bRv39/nJycWLduHX/5y1/Iy8t76DX0ej3Dhg0jKyuLuLi4VlXIOnXqxLJly+jcuTM1NTVkZmayZs0a1q9fT2lpaZ3neHp6Mm/ePCIiIvD09ESj0aDRaNi6dStz586lvLz8ofcVQtC+fXtcXV0pKioiPz+fioqKRuffz8+P6urqVnlmQgg8PDzo3bs3wcHBSJJEfHw8KSkp3Lp1i9LSUpvrqlKpxMPDg9GjRzNlyhR69+5Nfn4+n332GWvXrm3UgGby5MlERkYyZ86ch7ZRzYlCoaB79+4sXryYf/zjH/z444/1Klv+/v7Mnz+fiRMnsmnTJlavXk2XLl0YPXo0/fr1o6amhs2bN7Nz504yMjJaLM+PomBEA52FEE9gUiymATMaOkGhUDBkyBDKysrw8/NDoTCFgBgMBvz9/TEajdjZ2aFSqTAajSiVSgAuXbrE5s2bH9rp24rlvu7u7vTr1w9vb2+GDx9OQEAAxcXFrFy5kp07dzY4uqmpqWHVqlUAjbKcCCEYMGAAr732GqNGjaKsrIySkhJyc3M5f/48R44cISYmhpycHPLy8h5JW78fjUaDm5sbeXl5TW4cnJ2dGTFiBNevX39g5NRUs6lWq+WJJ55g8eLFjBgxgszMTKZOncrSpUtZu3Ztvc9WCIGPjw+vvvoqer2ee/fuceDAAZvLiEqlIiQkBB8fH1QqFb169WLx4sX86le/Iicnp9FyNBY3Nzeef/55nJ2dyc/P5+uvv7bpPGdnZ2bOnMncuXNxdnYmLS2N48ePYzQa8fb2Ztq0aRw7dozk5ORmU5TUajUuLi506NCB8vJyqqqqyMrKsqljqw+DwcDMmTNxcHDgzp07Np3j5OTE/Pnzee655+jYsSNGo9E6+uzevTvOzs4NKhhKpRKNRoOvry8vvfQSJSUlrF69mvPnz7dKh6lQKJgyZQrx8fG88847dOrUiRdffJFFixaRnZ3Nd999V+udKRQKOnbsyLx58xg4cCA//vgj6enpBAQEMG3aNPz8/LCzs2vwPSgUClxcXHjyyScZN24cwcHB5OXlER8fz5o1a0hPT6/33LoYM2YMxcXFNtez+toFlUrVYNsmhMDLy4s33niDF154ATs7O0pKSjAajdy6dYtvv/2WzZs3k5WV9dA2UqVSMXjwYJYsWUJISAglJSV8++23DBgwgF/96lfs27eP1NRUm+QBUzkKDAzE3t6+VRUMJycnpk6dSmRkJEePHiUqKqpO2RUKBYMGDWLKlCm4ubkxffp0nnnmGXx8fHBwcLD2f7169cLLy4ulS5c22Yr6MJqsYEiSVC2EeA3YByiBtZIkxTV4M5WKJUuWEBwcTG5uLomJieTk5ODo6MiYMWP45ptvyMnJITk5mXv37pGSkkJBQQEFBQXNplz079+f3r17U1lZyYwZMwgJCUGtVqNUKqmqqsLOzo7i4mIOHjzI3bt3G7xWU1wyDg4OvPDCC4wYMYLCwkJWrFhBSUkJAwcOpF+/ftb0a9eusWLFCqKjo5uloxg8eDDLly9n2bJl1oZMoVDg6OhIRUXFQzsLjUbDq6++yptvvsny5cu5evWq9RpPPvkkarWaqKioRuXJ3d2dWbNmMWbMGAD+8Y9/cPz4cVavXk1YWBhff/11g6MsvV6Pp6cnBoOB9u3bI0SdbsA6CQ4O5p133sFgMJCbm0u7du0YMGAAQUFBLa5gKJVK/Pz88Pf3R6lU0q5dO5sUNLVazYgRI5g7dy53795l1apVnDp1irg4U7WbOHEi//Vf/0VcXFyzKRcODg7Mnj2bqVOn4uPjQ3V1NeXl5axdu5ZVq1ZRVFTU6GvqdDoiIyMZPHgwJ0+e5NSpUw89R61WM2zYMGbPno2HhwcxMTHs37+fixcvotVquXLlSoNuFjs7O4KCgujWrRvdu3dnwIAB2NnZIYSgvLycixcvtri7RKFQ4OHhwTfffMOxY8c4fPgwBw8e5M9//jMzZsxg7969tawYnTt35r333iMyMtJqodPr9YSGhqLT6YiPj2+w3goh6Nq1K//7v/9LWFgYZ86cYf/+/UydOpUxY8Zw8+ZN1q9fb7PcFivZ1atXbTrHzc2NJ598krNnz1JQUGBNDwgI4LnnnmPTpk31vjMnJyeeffZZxo0bh4ODAwcOHCA6OprOnTvTo0cPpk6dikql4sCBA1y5cqXeDtLyDGbPnk1YWBh3795lx44dbN++nW7duhEYGNgkC3heXl6TLEBNRa1WM3jwYKvcDSlVrq6uTJgwAVdXV5RKJd7e3nh6elJVVcW1a9fQaDT4+fnh6OiIq6tro9rNxvJIMRiSJO0Gdtt6fGVlJb/85S/x8PAgLy+PwsJCysvLGT58OMHBwSxYsKDRoyJHR0d0Op1NxyqVSiZPnsycOXOoqqqiXbt2FBUVERUVxe3bt1Gr1YwfP5527dqh1WofqmDUxbBhw6ioqODMmTN17u/atSt9+/ZFoVBw6NAhNm7cSGFhIXv27KF79+54e3tbG73m9nG6ubnxy1/+kpiYGMrLy3n++ecZO3YssbGx/OUvf+HmzZsIIXBxccHBwYHMzExrxe3bty9z587FxcUFg8Fg7RDd3Nz47LPPSElJYdasWfWaeX+KEILhw4czf/58oqKiWLx4MXl5ecycORNnZ2cSEhIa1KotfnSFQkFBQQH5+fk2d6pOTk7MmzePwMBAdu3aRUxMDAsXLkSn0z2yu80WNBoNXbt2xdHRkXv37tl8nl6vZ9y4cbi4uPDGG2+wf//+Ws9ox44dHD16lLS0tGbJpxCCMWPG8Lvf/Y7U1FTOnDlDWVkZZWVlvP7667Rv354//elPja6z3t7ezJ49G41Gw4YNG2xya6jVavr06YOnpyeZmZksXLiQH3/80TrwaOjd63Q6+vXrx+zZsxk0aBAGgwFnZ2fUajVPPfUUUVFRxMfHN+pdNAVHR0e6devGhg0bAJMVNCkpieXLlzN//nxUqn81xyqVinHjxjFmzBhUKhXDhg0jNDQUIQTV1dVERUWxadOmBi0vPXr0YPHixYSHh7N9+3Y+//xzEhIScHFxoUePHnTo0AGFQmGzgmFvb4+7uzvr169/6LHdu3fnww8/JCcnh8uXL9eSa86cOURGRrJt27Y6z1UqlYSGhvLWW2+hUCj49NNPWblyJdnZ2eh0Ojp06ICfnx8BAQH07t2bjIyMOq1gQgiGDh3KkiVLGD58OMXFxSxevJiLFy8yffp0QkJC+PHHH7l9u/GefYPBgEajafR5TUEIQbdu3Xjrrbfw8/MjOTmZc+fO1avgDBw4kMjISNRqNXfv3iUpKYlDhw5x+fJloqOjeeaZZ1i0aBHOzs4tnvcWD/L8KVlZWWRlZdVKGzhwIGfOnGmyT7CqqsqmzkWpVFpHvEajkdOnT/Ppp59y5swZvL29mT9/PlVVVZw+fbpJIzM7OzvCw8MpLi7m/PnzD2iZCoWC8PBwOnbsSFlZGdu3bycjI4Oamhru3r1LcnJyLW2yOWMBoqOj+fLLL3njjTd4//33sbe3x9XVFTs7O/77v/+bgoIC1q5dy+DBg5kxYwaenp588sknfPPNNxiNRjp16oSrqysnT57k+vXrVuuNt7c33bp1IyEhoVHuHIv5V61Ws27dOmJjYxFCsGvXLjIyMqxm//pwcnKiV69e6PV6rl27RmZmpk0NpVKpZPz48YwdO5azZ8+ydOlSJk2ahEajoaioiNzcXJtlaCoGg4GuXbuiVCrJzc0lIyPDpnddVlbG5cuXmTJlCmFhYZw+fbqWEmqx9jUXGo3GqjAvWLCACxcuYDQa8fPzY+TIkUyYMIENGzYQGxtr8zUtz79fv34kJiZy8uRJmyyBkiRZ3TMGg4Hg4GAuXrxok3Li4eHBrFmzmDhxInq9nvz8fBITE+nSpQsODg44OTlZzcYtiVarRa/XP5Cu0+nIzs6uZaX19vZm0qRJpKSksHnzZlQqFXq9npqaGq5cucLRo0cb9J1rtVpefPFFwsPDOXHiBMuWLSM1NRW9Xk9AQACSJHHv3r1GtTHu7u507tz5gfb7p+j1en7zm9+Qn5/PokWLanX+YWFhTJs2ja+++qpet4ROp6Nnz554enqyb98+Nm7cSFpaGkajkXv37pGfn09SUhL29vbY2dnVGaRpcaHOnTuXkJAQFAqF1Y04bNgwIiMjuXnzJtu2bWtUnXFzc8Pf358bN260mgXD29ubOXPmMGjQICoqKvjqq69ITEyss97odDoGDRqEXq/HaDRy6NAh3nvvPS5fvkx5eTlCCMaPH49Go0GSpBafGNHqCsZP0Wq1hIWF8cknnzSpQ+3duzc3btzAxcWFwYMHk5WVxfnz5+u8loODA0FBQQghrJ360KFDGTt2LP3796esrIzly5fz9ddfN9onJYRgwoQJvPzyy1y+fJnVq1c/UHA1Gg2BgYFotVoyMzNJTk5+4AW3VIBheXk5MTExSJLEhAkTuHz5Mn/605/o2bMnS5YsYdSoUYSEhODl5YXBYMDd3d36rIQQ+Pr6UlhYyO9//3suXrwImDqLYcOGodVqSU9Pb1SFMxqNJCUlUVFRYZVZkiTi4uJITExs8PlblJMJEybQvn17q0XGlmcXEBDA66+/DsBHH31EXFwc06dPRwhBSkoKN27csFmGpuLi4kJQUBAKhYKKigqbrT4VFRXs3r2biIgI5syZQ35+Ph9//HGLNXTV1dWkp6fj4uJCz549OXXqFEajkezsbNLS0hg6dCjjxo0jPj7e5oZKr9dbR1dbtmyxOf6ioqKC7777jqCgICZMmMCiRYvo1KkTS5YsabCDsMxO6tChA1qtlpKSEo4fP87169fx9vZGpVJhZ2fXKqNRhUKBWq3G3d29VnpqaiqrVq2qZY0IDAykS5cu7N27lxUrVtQqIw971kIIAgMDiYiIoLCwkJUrV5KcnAyYOvh+/fpZZxQolcqHDgwsbYCdnZ213WyI0NBQhgwZwvz588nMzLSmK5VKXnrpJQD27NlT732NRiNlZWVUVlZy6dKlOttJSz7qy4uvry8LFy5k3LhxHDp0iNLSUiZOnMiiRYuoqKjg/PnzvPPOO5w+ffqh8tvZ2eHq6kp1dTWDBw/Gx8eH48ePt0rcjkqlom/fvowZMwaFQsHBgwfZtWtXvW2Gv78/Tz31FEqlkpSUFN5///1abnYhhHXmWWlpKTdu3GhR1+DPrmC4ublRUlJSr0vhYTg6OlJZWcmKFSsICAigqqqKX//613VG0QshrIGjarWaUaNGMWLECIqLi4mJieHTTz/l+PHjTbJeODg4MHHiRFxcXIiOjq5z+o9SqcTe3h6FQkFJSUmjZz08Co6OjkRERGBnZ0dcXByzZ8/mxo0bzJgxA5VKRZ8+fTh69CgLFy7kz3/+M1qtlhMnTlBTU4NCoaBDhw6Ayc1l6dB8fX0ZP368dXTQmEBPSZKIjY2luLiY0NBQfvjhB6slyhblzsnJCS8vL1QqlbWhsaXhHTt2LN27d+f48ePWKOxr166RlJTEunXrWmXqXUVFhXXU5eXlRadOnWx6dpIkkZKSwooVK+jYsSMzZsxg3759xMTEtEg+a2pq2Lt3L6+++ipTp05l//79JCcnU1xczLfffsvAgQPp0aMHOp3OZvfCkCFD6NGjBykpKezYscNmq5fRaCQlJYUPP/yQkpIS5syZw7PPPsuWLVs4efLkQ5+dQqGgsrKS6Ohovv76a3x9fampqUGv19OtWzfc3Nya/d1bZkH4+fkhhMDNzQ2tVstLL71ERUUFd+7cITMzk/z8/AeCLRUKhdWqYjQabSrblmegUCjo2rUrnp6e7N+/3zozSqlUMnToUDw9PVEqlfTt2xd/f3+uXbtW5zUt8VVDhw7F2dnZOvCYMWMGt2/fJi8vj/T09FpKBJjivXJych4ol76+vvTr148tW7bU2qfRaFAqldYOu6KigtTUVGpqavD398fFxcVmK58l3yNGjOAXv/gF9+7dY+PGjWRmZlqtb1evXmXdunWcP3/+oW7YoKAg5s6dy4gRI8jPz+f69etUVFRw6tSpZosLbIiuXbvy4osv4ubmxoULF3jvvfdISEio81kIIaxu9rKyMk6dOvXAsRqNhi5duqDRaLh9+zYJCQltV8FQKBRMnTqVO3fuNKlyq9VqHB0defbZZ0lOTmbhwoU8/fTTPPXUU3VG0ZeXl5OSkkL//v25e/cumzZtIj4+npiYGBISEsjJybFWZCEETzzxBAqFosEpf2q1mgEDBvDss8/Sv39/vvzyy3otIOXl5dy+fZvq6mokSWrVqagDBgxg4sSJpKamsmDBAi5fvsz48eN56qmnyM7OZv369Xz66af0798fX19f4uLiOH/+PGBq4BISEjAYDNZn27t3b95++21CQkIQQjBu3Dh27dpFdHS0zaPZzMxM4uLirHP7bVXslEolTk5O6HQ6SktLSUtLs6mDc3BwIDw8nLKyMlavXk1RURGSJLFlyxbOnDlDampqs83aaQhLfI0kSdY1LGzFaDRy7NgxPvroI5YvX85bb73FK6+8YrMVpLEkJSXx5Zdf8vrrr/P+++9brT5Hjhzhxo0bjBgxgrCwMPbv3//Q8qxUKomMjESv17Nu3TqbZzAolUq0Wi3l5eVcu3aN5cuX06tXL0JDQxk3bhwJCQkNurYkSaK6upqqqipu377NvXv3cHV1RaVSoVar8fLyahF/tIeHB5988gn9+pnWF3RwcKBdu3Z069aNsLAwioqKyMrK4s6dO6xbt47du3fXeoZCCOzt7a1rw9SFUqnE39+fkSNHsnXr1lp1qKamhpycHGuMjLOzM35+fty9e5fExET69+/PtGnTWLFiRZ2zISxWi6qqKoqKihg7dixXrlxBrVbj4eGBt7c3RqPxAQWjuroaR0dH3NzcuHfvHgaDgR49evA///M/eHt7s2/fPiRJwsXFhcDAQCZPnkxBQQHLli0D/uUOE0IwatQobty4wV//+lebY+LatWvHtGnT8PT0ZMuWLZw+fZqMjAxiYmLQ6XTWmXu2XOePf/wjkydPtq5VFBoailartbYdLYm7uzsvvPACo0aNoqCggFWrVnHu3Ll62yidTsekSZPw9vYmKiqK5cuXP9CmOjg44OvrixCCnTt32hRg/Sj8rAqGvb09zz//PL/73e+adL6joyO9evWivLycN998k7y8PE6fPs2oUaPqHBGWlZWxefNmQkJCcHd3p2fPniQlJVFVVYWzszNKpRK9Xo9Go6FPnz48/fTTfP/99/VO+bMEKn7yyScEBASwZ88eFi9e3KCy1JqLgd1PSkoKK1euJDY2lsOHD1NVVUVycjJffPEF0dHRHD58mIqKCgYPHoxer6esrKxW8N7Vq1fJzs7mtddeY8SIEXTv3p0OHTpw+PBhXFxc6NOnDx9//DErVqywRoZbZgnVR0lJCTt27GDevHkEBQVx9uxZm2RxdHRkwIABuLm5kZiYyOHDh21aLMfDw4OgoCBrA2t5pyUlJY2KI3hUDAYDnTt3RqFQUFpaalNjpVQqrT5TSZJIT0+nsrKSoKCgBjugR6W8vJzPP/8clUrFzJkz2bBhA5cuXbJOU3Zzc+Ott95Cq9WSnZ1NcXExCQkJ9fqH/f39KS4u5vDhwzaNAFUqFRMnTmT06NFERUURHR1Nfn4+UVFRDBkyhEmTJnHy5El27dpV5zOUJImioiIuXbpE7969GTNmDL1798bR0REnJ6daxzU3d+/e5fvvv+eHH36gvLycnj17MmjQILp3787WrVtJS0ujpqaG9PR0bt68WSsPBQUFlJSUEBoaypQpU9iwYYNViRZCoNFocHR05Be/+AUvvPACWq2W3bt3W8vSrVu3KCsrIzQ0lOHDh3P16lXCw8MZPnw4Bw4c4LPPPuPtt99m8uTJpKens2fPngcWKqupqeHEiROcOHEChULBgAED2L17N5s2bWpQ7m3btjFy5Eg++OADsrKy8PX1pUOHDsTHx3Pnzh3mzZvHlClTrIGaOTk5fP7557WuUV1dTXV1NW5ubvTv3x87OzubFQxHR0cCAgIAUxtUWFiI0Wjk7t273Lt3D71eT2BgIDqdjpSUlHrrzrBhwxg7dix5eXm888477Nu3j6effpoPPvgAFxcXm/LSVIQQjBw5khkzZqDVajl8+DDHjh1rcABUU1PD2bNn0ev1bNu2jevXrz9Qrrt160anTp2oqakhLS2txdoNCw9VMIQQvsB6wB2QgC8kSfpYCPEnYDZg6UH+YJ5VYjNhYWHk5+dbp9k1Fjc3N3x8fHj33XetnbrFxF5fY3Po0CGWLl3KjBkz6NWrFwsWLCA9PZ27d++Sm5uLq6srYDIlWaJ167qWQqGgV69e/PGPf8THx4fdu3fz3nvvtdrqdo3lxo0bvP3227UsJ5cuXeLy5cu1rDbff/+9da2L+wtzVFQUr7zyCh9++CHDhg0jIyODDz74gL///e+4u7uzbNkyIiIiWL9+PUajkevXr/Pyyy/XUjDuH6lbfIH5+fkYDAa8vb1tlsWyuJoQgn379nHlyhWbLA9arRZJkigpKaGqqsraaVtoLeXPYv6WJIlr165x6dKleo9VKpUEBwczcuRISkpKyM/Px8/Pj0mTJqHVajly5EiL+4Jzc3N57733OHPmDM899xyBgYH07duX7OxsNm7cSHh4OKtWreLevXvs3Lmz3tlgnTt3pk+fPmRlZZGWlmbzqptubm6MGDGC6dOnW83U2dnZVFRU4OvrS2BgYIMuptzcXLZu3YqjoyPjxo0jMDAQjUZjHeE3djVHWykpKeGrr76ybm/atIn+/fvzySefsGXLFk6dOlVvnpOTkzl27BjPP/88v/3tb/H19SU2NpaamhrUajU9e/bEx8eH4cOH4+7uzvbt262dhdFoJDY2luPHjzNx4kRWr15NUlISvr6+5Obmsn79es6ePcs///lPfvvb3/LOO++g1+tZvXp1vWXJz88Pg8HAiRMnHip3XFwcs2bNYvTo0djZ2XHy5EmOHj3KrVu36N69O7NmzbKmr1y5kqNHj9ayKEiSxO3bt7l69SoDBgzA3d0dvV5Pdna2Tc+9qKiIjIwMAgMD8fX1JTIy0hoU2a9fP5555hl69+5NWVkZixYtYt++fXVep0OHDuh0Or788ks2btyIJEmoVCqry9iyCGRLoNVqGTx4sHUQ9be//e2hs8MqKyutC4eVlZU90J6pVCoiIiJwd3cnKyuLs2fPtrjF1hYLRjXwpiRJF4QQjsB5IcQB876/SpL0UVNvbjQaWbt2bZMj9/V6PaWlpezZs8daUdVqdYMjo+LiYtauXcuuXbsICwtjwIABODg44OXlRWlpKeXl5Zw4cYIff/yRW7du1Wu279KlCx9//DFdunTh3XffZd26dTZVAItloLKystWtGXXd7/40SZKIiopi0qRJD1gwqqurOXjwINOnT8fX15fr169bAzsLCwv5zW9+w9ixY/H29qayspL9+/dz4cIF6/ldu3YlPDzcGgOjUqkICgqiY8eOFBUVPWBmfZgcZWVl5ObmcvXqVXJycmzqrK5fv87u3bt5+eWXef/99zl58iSFhYVotVoqKyv5/vvvG5WPplJQUMC1a9cYNGgQiYmJDc6EMBgMLF68mNGjR1NdXU1FRQVqtZqqqiq2bdv2QHBgS1FRUcHevXs5cuQIekQsphoAAAtCSURBVL2e9u3bU1paSmFhIT179iQwMJDCwkLOnTtXp3KhUCgIDQ2lffv2bNmyxWb3SFVVFVu2bKGoqIiZM2cSFhaGj48PlZWVVgW1uLi4wfdfXV1NYmIiX3zxBcnJyUyePJkOHTqQnJxMdHQ0hw4davSCU03l5s2b1imoUVFR9XZQBQUFfPzxx3h6ehISEsKCBQtquV3VarU1CHfNmjV89tlntdwcpaWlLF26lMuXLxMREUFFRQV79uzh4MGDxMXFUV1dzYYNGzh79iw+Pj7ExsY2GCwcEBCAUql86AwSC6mpqaxZs8aq+Fnez+XLl1mwYAFgei8Wi9xPyczMZOfOnXh7e2Nvb2/zUgRgKqvp6elIksRzzz1HeHi41f3p5eWFh4cHarWaM2fONKhYpqSkkJubi5+fH2+++aZ19Vy1Wo2vr2+LKhgKhcK6gOCRI0e4cuWKTW2cxfJTFzqdjuDgYJRKJQkJCa2y2u9DFQxJkjKBTPP/d4UQVzF96OyROXDgwMMPagCtVktaWlotq8GTTz5Za2ZCXViW6P3222/ZsWMHarUag8FAeXk5CoWC/Pz8BjU7lUrFrFmz6NmzJ8uWLeNvf/ubTWsBGI1GDhw4gMFgIDs7u8XXgW8K1dXV9c4Lr66uJjY29gF3gsVkv3LlSus02/ufv1KpJCIigj/84Q8YjUZSU1PJy8sjOjqac+fOcebMmUZZsUpLS9m1axdXr17l/PnzNlfyyspK/u///o+QkBAiIyMZM2aM1ZKRk5Nj7cxaej2EO3fusHHjRrRaLTt37mzQjVRRUUFsbKxVcc7KyuLixYtcuXKFpKSkFhl514fFN15eXl5rUHDq1CmbllgvKysjKiqKb775plEBcrm5uWzevJnTp08TERFBWFgYwcHB5OTksHXrVnbu3PnQe1sWGcrOziY+Ph69Xs+NGzfIysqiqKioxVYy/Cn5+flcvHgRHx8f6/dQ6iMmJobFixfz2muvERoaarWu5ubmkpKSQlJSErt37+b48eMUFhY+8Axu3LjB559/zrZt2zAajWRlZdVSIkpKSrhw4UKtgUBdCCHo3bt3rXVHbKG+ODNbZj0VFxezZs0arl27hr29faMUwLKyMlauXEliYiKjRo3Cy8sLd3d3cnJy2LFjB6dPn0ar1XLx4sUGV/A8deoUH330EXPnziUyMhKtVsutW7dYt24da9asadEyU15ezs6dOzl37hx79uxplunnZWVl/PDDD/j7+3P48OEmTWZoLKIxvkchRABwHAgG3gB+BRQD5zBZOR54Ci35NdXhw4fTtWtXvvjiC2tBjoiI4Pbt2012u9iCEIKIiAj8/f355z//adNc/PvPtZjqW3MluJ+bTp06ERwcTGVlpdX3l5ubax2VN8UH3pTlyZVKJZ07d2bcuHF4enoCJsXvypUrHD582Ob1NB4Vy8fubPl4k06ns3ZGlmDFx/FbNa6urhgMBlJSUppsvVMoFLi6uuLn50deXh63bt1qlcDc5iQ4OBgPDw8OHz5s03NwcXHhiSeewMfHBzBZQTIyMiguLm5y3WkMCoWC5cuX89VXXzXozmsJNBoNCoWiScvSK5VKfH19cXNzQwhhnfXSmHZXq9USHBxMly5d6Ny5M0ePHuXcuXMtPggBU4yiEKLR65U0hCX+JDMz02ZrlC3U9zVVmxUMIYQeOAb8P0mSvhNCuAO5mOIylgKekiTNesg1mrUmWD6G1hrThepC/mzx44tlbr+Fnyv4VkbmcUCn07WKMvPvjNze188jKRhCCDWwC9gnSdJf6tgfAOySJCn4Ide5CyTakN/HlfaYlK62iizf440s3+NPW5dRlu/xw1+SpA517bBlFokA1gBX71cuhBCe5vgMgImALfP8EiVJ6m/DcY8lQohzsnyPL7J8jzdtXT5o+zLK8rUtbJlFMhT4JXBFCGFxwP0BmC6E6IPJRZIKzGmRHMrIyMjIyMg8dtgyi+QEUJd/pVFrXsjIyMjIyMj859DynxCszRetfL/WRpbv8UaW7/GmrcsHbV9GWb42RKOmqcrIyMjIyMjI2EJrWzBkZGRkZGRk/gOQFQwZGRkZGRmZZqfVFAwhxBghRKIQIkkIsbC17tucCCHWCiGyhRCx96W5CCEOCCGum/+2M6cLIcTfzPJeFkL0/flybhtCCF8hxBEhRLwQIk4I8bo5vU3IKITQCSHOCiFizPK9bU5/QghxxizHFiGExpyuNW8nmfcH/Jz5twUhhFIIcVEIscu83WZkAxBCpAohrgghLgkhzpnT2kT5BBBCGIQQ3wohEoQQV4UQIW1FPiFEkPm9WX7FQoj5bUU+ACHEb81tS6wQYpO5zWlTdbAxtIqCIYRQAn8Hnga6Y5ri2r017t3MrAPG/CRtIXBIkqTOwCHzNphk7Wz+/Rr4rJXy+ChYPmzXHRgMzDW/p7YiYwUwUpKk3kAfYIwQYjDwPqYP9wUCBcDL5uNfBgrM6X81H/fvzuvA1fu225JsFp6SJKnPfesJtJXyCfAxsFeSpK5Ab0zvsk3IJ0lSovm99cH02Yh7wDbaiHxCCG9gHtDfvOikEphG26yDtmH5IE1L/oAQTKuAWrYXAYta494tIEsAEHvfdiKmZdIBPDEtJgawEphe13GPyw/YAYS3RRkBe+ACMAjTynoqc7q1rAL7gBDz/yrzceLnznsDMvlgaqBHYlp5V7QV2e6TMRVo/5O0NlE+AWcg5afvoa3I9xOZIoCTbUk+TB8BTQdczHVqFxDZ1upgY36t5SKxPHgLt2imL7L+G+Au/WtF0zuAu/n/x1pms7nuSeAMbUhGswvhEpANHACSgUJJkixfzLpfBqt85v1FgGvr5rhRrAB+D1g+rOJK25HNggTsF0KcF6YPKULbKZ9PADnAl2Y312ohhANtR777mQZsMv/fJuSTJOk28BGQhukL5EXAedpeHbQZOcizGZFMquhjP+9XmD5s909gviRJxffve9xllCSpRjKZaH2AgUDXnzlLzYIQYhyQLUnS+Z87Ly1MqCRJfTGZz+cKIYbdv/MxL58qoC/wmSRJTwKl/MtdADz28gFgjkEYD2z96b7HWT5z7MgETIqiF+DAgy71/yhaS8G4Dfjet+1jTmsLZAkhPMH0fRZMI2N4TGUWpg/b/RPYKEnSd+bkNiUjgCRJhcARTCZLgxDCsqrt/TJY5TPvdwbyWjmrtjIUGC+ESAU2Y3KTfEzbkM2KeZSIJEnZmPz3A2k75fMWcEuSpDPm7W8xKRxtRT4LTwMXJEmyfC+8rcg3GkiRJClHkqQq4DtM9bJN1cHG0FoKRjTQ2RxNq8FkHtvZSvduaXYCM83/z8QUt2BJf9EcCT0YKLrPDPhviRB1f9iONiKjEKKDEMJg/t8OU3zJVUyKxhTzYT+VzyL3FOCweYT1b4ckSYskSfKRJCkAU/06LEnSC7QB2SwIIRyEEI6W/zH58WNpI+VTkqQ7QLoQIsicNAqIp43Idx/T+Zd7BNqOfGnAYCGEvbkttby/NlMHG01rBXsAY4FrmHzei3/u4JMmyrAJk2+tCtNo42VMPrNDwHXgIOBiPlZgmjmTDFzBFFn8s8vwEPlCMZknLwOXzL+xbUVGoBdw0SxfLLDEnN4ROAskYTLbas3pOvN2knl/x59bBhvlHAHsamuymWWJMf/iLO1IWymf5jz3Ac6Zy+h2oF0bk88B0yjd+b60tiTf20CCuX35GtC2pTrY2J+8VLiMjIyMjIxMsyMHecrIyMjIyMg0O7KCISMjIyMjI9PsyAqGjIyMjIyMTLMjKxgyMjIyMjIyzY6sYMjIyMjIyMg0O7KCISMjIyMjI9PsyAqGjIyMjIyMTLPz/wEXbplnmlgGHwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 648x144 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VmgFAVAFr_1u",
        "colab_type": "text"
      },
      "source": [
        "#### Save Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qiEyviXGa32C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Now to save these models for easy loading\n",
        "pp = pathlib.Path('.') / 'sentences'\n",
        "pp.mkdir(exist_ok=True)  # create the directory\n",
        "\n",
        "np.save(pp / 'train_images', train_images)\n",
        "np.save(pp / 'test_images', test_images)\n",
        "np.save(pp / 'train_labels', train_labels)\n",
        "np.save(pp / 'test_labels', test_labels)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j8kQvdfl-Ox1",
        "colab_type": "text"
      },
      "source": [
        "# Prepare Training Dataset(Label)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qDw__t1Jlk-8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create a dictionary of mapping from \n",
        "mappings = dict(zip(np.arange(0, 47), MAPPINGS))\n",
        "# add a space for spacing words\n",
        "mappings[47] = ' '\n",
        "# inverse: character as key and value as it's integer encoding\n",
        "inverse_mappings = {v: k for k, v in mappings.items()}"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SxCE4CgilaPA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        },
        "outputId": "e64fb84c-4655-446c-ec03-770b5e01b6f8"
      },
      "source": [
        "# now convert categorical labels from the sentences\n",
        "encode = lambda x: [inverse_mappings[xi] if xi in inverse_mappings else inverse_mappings[xi.upper()] for xi in x]\n",
        "decode = lambda x: [mappings[xi] for xi in x]\n",
        "\n",
        "train_labels_cat = np.array([encode(xi) for xi in train_labels])\n",
        "test_labels_cat = np.array([encode(xi) for xi in test_labels])\n",
        "\n",
        "# Verify the encoding/decoding\n",
        "print(train_labels[1])\n",
        "print(train_labels_cat.shape, train_labels_cat[1])\n",
        "print(decode(train_labels_cat[1]))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ABS CBN also utilized what is pr\n",
            "(4000, 32) [10 11 28 47 12 11 23 47 36 21 28 24 47 30 46 18 21 18 35 39 38 47 32 42\n",
            " 36 46 47 18 28 47 25 45]\n",
            "['A', 'B', 'S', ' ', 'C', 'B', 'N', ' ', 'a', 'L', 'S', 'O', ' ', 'U', 't', 'I', 'L', 'I', 'Z', 'e', 'd', ' ', 'W', 'h', 'a', 't', ' ', 'I', 'S', ' ', 'P', 'r']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "195rY50uPhrw",
        "colab_type": "text"
      },
      "source": [
        "# Build Train and Test Loader\n",
        "\n",
        "Using `tf.data`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VKNx2nN4Pqcc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "AUTO = tf.data.experimental.AUTOTUNE\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "# This step will make sense after looking at the model\n",
        "def merge_image_label(image, label):\n",
        "    return (image, label), label\n",
        "\n",
        "trainloader = tf.data.Dataset.from_tensor_slices((train_images, train_labels_cat))\n",
        "testloader = tf.data.Dataset.from_tensor_slices((test_images, test_labels_cat))\n",
        "\n",
        "trainloader = (\n",
        "    trainloader\n",
        "    .shuffle(1024)\n",
        "    .map(merge_image_label, num_parallel_calls=AUTO)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(AUTO)\n",
        ")\n",
        "\n",
        "testloader = (\n",
        "    testloader\n",
        "    .map(merge_image_label, num_parallel_calls=AUTO)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(AUTO)\n",
        ")"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xg-LdXSDsbPs",
        "colab_type": "text"
      },
      "source": [
        "# Build Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5fdP1t6zuU73",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# CNN definition : LeNet5 architecture, a classic architecture\n",
        "IMG_H, IMG_W = train_images[1].shape\n",
        "# we limited output sentences to 32 chars\n",
        "OUTPUT_LENGTH = 32  \n",
        "# EMNIST has 47 labels, adding one for 'blank space'\n",
        "NUM_CLASSES = 49 # notice two additional class; one is for \"space\" another is required by \"CTCLayer\""
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XNeF-P0gsit7",
        "colab_type": "text"
      },
      "source": [
        "#### Utility to Extract Patch\n",
        "\n",
        "Since each image is a sentence, we will extract patches from it\n",
        "width of each patch will be 20 pixels wide (note EMNIST is 28 px)\n",
        "and every time we will move 14 px over to extract another sample"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YfI1fixxumPn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "PATCH_WIDTH = 20\n",
        "PATCH_STRIDE = 14"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RExyW0Opi_g_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def extract_patches(image):\n",
        "    kernel = [1, 1, PATCH_WIDTH, 1]\n",
        "    strides = [1, 1, PATCH_STRIDE, 1]\n",
        "    patches = tf.image.extract_patches(image, kernel, strides, [1, 1, 1, 1], 'VALID')\n",
        "    patches = tf.transpose(patches, (0, 2, 1, 3))\n",
        "    patches = tf.expand_dims(patches, -1)\n",
        "    return patches"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wmjDDWLfHWyJ",
        "colab_type": "text"
      },
      "source": [
        "#### CNN backbone\n",
        "\n",
        "* Takes in a patch of image\n",
        "* Returned flattened embedding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ra0ehhwMG3FF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        },
        "outputId": "125f67f1-1763-40a4-8c39-325351819f1f"
      },
      "source": [
        "def ImagePatchEncoder():\n",
        "  patched_inputs = Input(shape=(IMG_H, PATCH_WIDTH, 1))\n",
        "  x = Conv2D(32, kernel_size=(3, 3), activation='relu')(patched_inputs)\n",
        "  x = Conv2D(64, kernel_size=(3, 3), activation='relu')(x)\n",
        "  x = MaxPooling2D(pool_size=(2, 2))(x)\n",
        "  x = Dropout(0.2)(x)\n",
        "  flattened_outputs = Flatten()(x)\n",
        "\n",
        "  return Model(inputs=patched_inputs, outputs=flattened_outputs, name='backbone')\n",
        "\n",
        "tf.keras.backend.clear_session()\n",
        "image_patch_encoder = ImagePatchEncoder()\n",
        "image_patch_encoder.summary()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"backbone\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 28, 20, 1)]       0         \n",
            "_________________________________________________________________\n",
            "conv2d (Conv2D)              (None, 26, 18, 32)        320       \n",
            "_________________________________________________________________\n",
            "conv2d_1 (Conv2D)            (None, 24, 16, 64)        18496     \n",
            "_________________________________________________________________\n",
            "max_pooling2d (MaxPooling2D) (None, 12, 8, 64)         0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 12, 8, 64)         0         \n",
            "_________________________________________________________________\n",
            "flatten (Flatten)            (None, 6144)              0         \n",
            "=================================================================\n",
            "Total params: 18,816\n",
            "Trainable params: 18,816\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ezbWeXARO_C0",
        "colab_type": "text"
      },
      "source": [
        "#### CTC Loss Layer\n",
        "\n",
        "You can learn more about CTC Loss in this [awesome blog post](https://app.wandb.ai/authors/text-recognition-crnn-ctc/reports/Text-Recognition-with-CRNN-CTC-Network--VmlldzoxNTI5NDI)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MzyiMxNoH_vt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## Ref: https://keras.io/examples/vision/captcha_ocr/\n",
        "class CTCLayer(Layer):\n",
        "  def __init__(self, name=None):\n",
        "\n",
        "      super().__init__(name=name)\n",
        "      self.loss_fn = tf.keras.backend.ctc_batch_cost\n",
        "\n",
        "  def call(self, y_true, y_pred):\n",
        "      # Compute the training-time loss value and add it\n",
        "      # to the layer using `self.add_loss()`.\n",
        "\n",
        "      batch_len = tf.cast(tf.shape(y_true)[0], dtype=\"int64\")\n",
        "      input_length = tf.cast(tf.shape(y_pred)[1], dtype=\"int64\")\n",
        "      label_length = tf.cast(tf.shape(y_true)[1], dtype=\"int64\")\n",
        "\n",
        "      input_length = input_length * tf.ones(shape=(batch_len, 1), dtype=\"int64\")\n",
        "      label_length = label_length * tf.ones(shape=(batch_len, 1), dtype=\"int64\")\n",
        "\n",
        "      loss = self.loss_fn(y_true, y_pred, input_length, label_length)\n",
        "      self.add_loss(loss)\n",
        "\n",
        "      # At test time, just return the computed predictions\n",
        "      return y_pred\n",
        "      "
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Uun36GxI9G6V",
        "colab_type": "text"
      },
      "source": [
        "#### Final Model\n",
        "\n",
        "##### Image Encoder\n",
        "\n",
        "* Take in image input\n",
        "* Get patches of shape `(PATCH_HEIGHT, PATCH_WIDTH, 1)`.\n",
        "* Feed patches to backbone CNN architecture to get embedding. \n",
        "* Wrap using `TimeDistributed`\n",
        "\n",
        "\n",
        "##### Recurrent Network\n",
        "\n",
        "* Takes in `TimeDistributed` sequence\n",
        "* Computes output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2tJu9Yn5NNIt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ImagetoSequence():\n",
        "  # IMAGE ENCODER #\n",
        "  labels = Input(name=\"label\", shape=(None,), dtype=\"float32\")\n",
        "  image_input = Input(shape=(IMG_H, IMG_W), name=\"cnn_input\")\n",
        "  # reshape to add dimensions\n",
        "  image_reshaped = Reshape((IMG_H, IMG_W, 1))(image_input)\n",
        "  # extract patches of images\n",
        "  image_patches = Lambda(extract_patches)(image_reshaped)\n",
        "  # get CNN backbone architecture to get embedding for each patch\n",
        "  image_patch_encoder = ImagePatchEncoder()\n",
        "  # Wrapper allows to apply a layer to every temporal slice of an input.\n",
        "  time_wrapper = TimeDistributed(image_patch_encoder)(image_patches)\n",
        "\n",
        "  # RECURRENT NETWORK #\n",
        "  lstm_out = LSTM(128, return_sequences=True, name=\"lstm\")(time_wrapper)\n",
        "  softmax_output = Dense(NUM_CLASSES, activation='softmax', name=\"lstm_softmax\")(lstm_out)\n",
        "  # \n",
        "  output = CTCLayer(name=\"ctc_loss\")(labels, softmax_output)\n",
        "\n",
        "  return Model([image_input, labels], output)"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AFEYPgl3Qihf",
        "colab_type": "text"
      },
      "source": [
        "# Initialize Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WFnnEeoKIrYC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 479
        },
        "outputId": "48985c45-5d87-4e0e-cece-815eb368c90c"
      },
      "source": [
        "tf.keras.backend.clear_session()\n",
        "model = ImagetoSequence()\n",
        "model.summary()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "cnn_input (InputLayer)          [(None, 28, 896)]    0                                            \n",
            "__________________________________________________________________________________________________\n",
            "reshape (Reshape)               (None, 28, 896, 1)   0           cnn_input[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lambda (Lambda)                 (None, 63, 28, 20, 1 0           reshape[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed (TimeDistribut (None, 63, 6144)     18816       lambda[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "lstm (LSTM)                     (None, 63, 128)      3211776     time_distributed[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "label (InputLayer)              [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lstm_softmax (Dense)            (None, 63, 49)       6321        lstm[0][0]                       \n",
            "__________________________________________________________________________________________________\n",
            "ctc_loss (CTCLayer)             (None, 63, 49)       0           label[0][0]                      \n",
            "                                                                 lstm_softmax[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 3,236,913\n",
            "Trainable params: 3,236,913\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "asdJgVutPOsD",
        "colab_type": "text"
      },
      "source": [
        "# Compile Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LAuhDm8jm-Vg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# compile the models\n",
        "model.compile(tf.keras.optimizers.Adam())"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QeR-dUwePUEf",
        "colab_type": "text"
      },
      "source": [
        "# Train Model with W&B"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ueJ3cunufae5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "early_stopper = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss', min_delta=0, patience=10, verbose=0, mode='auto', restore_best_weights=True)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u4mQQ7i5nRU5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "7b13d37a-2622-4cba-bba7-214cf4c791b8"
      },
      "source": [
        "wandb.init(entity='iit-bhu', project='emnist-nlp')\n",
        "\n",
        "_ = model.fit(trainloader,  \n",
        "                    validation_data=testloader,\n",
        "                    epochs=100,\n",
        "                    callbacks=[WandbCallback(),\n",
        "                               early_stopper])"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/iit-bhu/emnist-nlp\" target=\"_blank\">https://app.wandb.ai/iit-bhu/emnist-nlp</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/iit-bhu/emnist-nlp/runs/1yyuw016\" target=\"_blank\">https://app.wandb.ai/iit-bhu/emnist-nlp/runs/1yyuw016</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "63/63 [==============================] - 12s 187ms/step - loss: 92.6480 - val_loss: 61.4961\n",
            "Epoch 2/100\n",
            "63/63 [==============================] - 11s 176ms/step - loss: 44.4361 - val_loss: 34.0389\n",
            "Epoch 3/100\n",
            "63/63 [==============================] - 11s 176ms/step - loss: 26.2492 - val_loss: 23.0972\n",
            "Epoch 4/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 18.8585 - val_loss: 18.5460\n",
            "Epoch 5/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 15.4288 - val_loss: 15.9215\n",
            "Epoch 6/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 13.3450 - val_loss: 14.2298\n",
            "Epoch 7/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 11.8815 - val_loss: 13.0436\n",
            "Epoch 8/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 10.7958 - val_loss: 12.1949\n",
            "Epoch 9/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 9.9511 - val_loss: 11.3838\n",
            "Epoch 10/100\n",
            "63/63 [==============================] - 11s 173ms/step - loss: 9.2488 - val_loss: 10.8165\n",
            "Epoch 11/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 8.6577 - val_loss: 10.3874\n",
            "Epoch 12/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 8.1245 - val_loss: 9.8470\n",
            "Epoch 13/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 7.6636 - val_loss: 9.5777\n",
            "Epoch 14/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 7.2568 - val_loss: 9.4508\n",
            "Epoch 15/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 6.8841 - val_loss: 8.9687\n",
            "Epoch 16/100\n",
            "63/63 [==============================] - 11s 173ms/step - loss: 6.5519 - val_loss: 8.6401\n",
            "Epoch 17/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 6.1909 - val_loss: 8.5145\n",
            "Epoch 18/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 5.9542 - val_loss: 8.2268\n",
            "Epoch 19/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 5.6795 - val_loss: 8.1211\n",
            "Epoch 20/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 5.4543 - val_loss: 7.9257\n",
            "Epoch 21/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 5.1840 - val_loss: 7.7784\n",
            "Epoch 22/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 5.0004 - val_loss: 7.7819\n",
            "Epoch 23/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 4.7747 - val_loss: 7.5118\n",
            "Epoch 24/100\n",
            "63/63 [==============================] - 11s 173ms/step - loss: 4.5795 - val_loss: 7.3938\n",
            "Epoch 25/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 4.4050 - val_loss: 7.2196\n",
            "Epoch 26/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 4.2361 - val_loss: 7.1990\n",
            "Epoch 27/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 4.0192 - val_loss: 7.0780\n",
            "Epoch 28/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 3.8700 - val_loss: 6.9571\n",
            "Epoch 29/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 3.7028 - val_loss: 6.9109\n",
            "Epoch 30/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 3.5485 - val_loss: 6.9479\n",
            "Epoch 31/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 3.4510 - val_loss: 6.9567\n",
            "Epoch 32/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 3.3122 - val_loss: 6.7464\n",
            "Epoch 33/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 3.1483 - val_loss: 6.7091\n",
            "Epoch 34/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 2.9870 - val_loss: 6.6150\n",
            "Epoch 35/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 2.8695 - val_loss: 6.5958\n",
            "Epoch 36/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 2.7608 - val_loss: 6.5102\n",
            "Epoch 37/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 2.6536 - val_loss: 6.4375\n",
            "Epoch 38/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 2.5335 - val_loss: 6.4990\n",
            "Epoch 39/100\n",
            "63/63 [==============================] - 11s 169ms/step - loss: 2.4280 - val_loss: 6.4508\n",
            "Epoch 40/100\n",
            "63/63 [==============================] - 11s 169ms/step - loss: 2.3440 - val_loss: 6.4760\n",
            "Epoch 41/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 2.2653 - val_loss: 6.4661\n",
            "Epoch 42/100\n",
            "63/63 [==============================] - 11s 169ms/step - loss: 2.1405 - val_loss: 6.4941\n",
            "Epoch 43/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 2.0545 - val_loss: 6.3958\n",
            "Epoch 44/100\n",
            "63/63 [==============================] - 11s 171ms/step - loss: 1.9871 - val_loss: 6.3797\n",
            "Epoch 45/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 1.8962 - val_loss: 6.4305\n",
            "Epoch 46/100\n",
            "63/63 [==============================] - 11s 169ms/step - loss: 1.8178 - val_loss: 6.3858\n",
            "Epoch 47/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 1.7406 - val_loss: 6.4080\n",
            "Epoch 48/100\n",
            "63/63 [==============================] - 11s 170ms/step - loss: 1.6547 - val_loss: 6.3528\n",
            "Epoch 49/100\n",
            "63/63 [==============================] - 11s 169ms/step - loss: 1.5930 - val_loss: 6.4405\n",
            "Epoch 50/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 1.5097 - val_loss: 6.3862\n",
            "Epoch 51/100\n",
            "63/63 [==============================] - 11s 172ms/step - loss: 1.4544 - val_loss: 6.3043\n",
            "Epoch 52/100\n",
            "63/63 [==============================] - 11s 169ms/step - loss: 1.3973 - val_loss: 6.3367\n",
            "Epoch 53/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 1.3157 - val_loss: 6.3581\n",
            "Epoch 54/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 1.2653 - val_loss: 6.5315\n",
            "Epoch 55/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 1.2076 - val_loss: 6.4651\n",
            "Epoch 56/100\n",
            "63/63 [==============================] - 11s 167ms/step - loss: 1.1373 - val_loss: 6.4773\n",
            "Epoch 57/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 1.0873 - val_loss: 6.5429\n",
            "Epoch 58/100\n",
            "63/63 [==============================] - 11s 169ms/step - loss: 1.0453 - val_loss: 6.5902\n",
            "Epoch 59/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 0.9911 - val_loss: 6.5197\n",
            "Epoch 60/100\n",
            "63/63 [==============================] - 11s 167ms/step - loss: 0.9331 - val_loss: 6.4398\n",
            "Epoch 61/100\n",
            "63/63 [==============================] - 11s 168ms/step - loss: 0.8886 - val_loss: 6.5362\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RbOx_WuMtYem",
        "colab_type": "text"
      },
      "source": [
        "#### Save your hard work"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QG6ooSDFfqcE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.save('emnist-nlp.h5')"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UngLxdFvtiXt",
        "colab_type": "text"
      },
      "source": [
        "# Prediction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Veszg_ejudSQ",
        "colab_type": "text"
      },
      "source": [
        "#### Simplify Model for Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RwpYiNI_uf-i",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386
        },
        "outputId": "c8992f69-5a2d-4cff-9935-cb90e2985c3e"
      },
      "source": [
        "# notice we are removing CTCLayer\n",
        "inference_model = Model(model.get_layer('cnn_input').input, model.get_layer('lstm_softmax').output)\n",
        "inference_model.summary()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"functional_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "cnn_input (InputLayer)       [(None, 28, 896)]         0         \n",
            "_________________________________________________________________\n",
            "reshape (Reshape)            (None, 28, 896, 1)        0         \n",
            "_________________________________________________________________\n",
            "lambda (Lambda)              (None, 63, 28, 20, 1)     0         \n",
            "_________________________________________________________________\n",
            "time_distributed (TimeDistri (None, 63, 6144)          18816     \n",
            "_________________________________________________________________\n",
            "lstm (LSTM)                  (None, 63, 128)           3211776   \n",
            "_________________________________________________________________\n",
            "lstm_softmax (Dense)         (None, 63, 49)            6321      \n",
            "=================================================================\n",
            "Total params: 3,236,913\n",
            "Trainable params: 3,236,913\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dBZC4UPhwQKg",
        "colab_type": "text"
      },
      "source": [
        "#### Get Predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vFyStXxLvI5X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "(val_images, val_labels), _ = next(iter(testloader))"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SMQt-H77wwH9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_prediction(val_images, val_labels):\n",
        "    # get prediction probability \n",
        "    predictions = inference_model.predict(val_images)\n",
        "    # get argmax\n",
        "    pred_indices = np.argmax(predictions, axis=2)\n",
        "    \n",
        "    actual_text_list = []\n",
        "    pred_text_list = []\n",
        "\n",
        "    for i in range(val_images.shape[0]):\n",
        "        ans = \"\"\n",
        "        pred_ans = \"\"\n",
        "\n",
        "        # get actual text\n",
        "        for p in val_labels[i].numpy():\n",
        "            if p in mappings.keys():\n",
        "              ans+=mappings[p]\n",
        "\n",
        "        # get predicted text from image\n",
        "        merged_list = [k for k,_ in groupby(pred_indices[i])]\n",
        "        for p in merged_list:\n",
        "            if p in mappings.keys():\n",
        "              pred_ans+=mappings[p]\n",
        "\n",
        "        actual_text_list.append(ans)\n",
        "        pred_text_list.append(pred_ans)\n",
        "\n",
        "    return actual_text_list, pred_text_list"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7k2LlgTP1TuI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "actual_text, predicted_text = get_prediction(val_images, val_labels)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3NfaOBJ6Yiub",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "table = wandb.Table(columns=[\"Actual Text\", \"Predicted Text\"])\n",
        "for act_text, pred_text in zip(actual_text, predicted_text):\n",
        "    table.add_data(act_text, pred_text)\n",
        "\n",
        "wandb.log({\"emnist-nlp\": table})"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wzSdAkt4d_xB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "wandb.log({\"images\": [wandb.Image(np.expand_dims(image, axis=2))\n",
        "                            for image in val_images]})"
      ],
      "execution_count": 41,
      "outputs": []
    }
  ]
}